From 403b5671088a7e1998e749af2578a4be72d3be94 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 14 Jun 2026 00:20:12 -0400 Subject: [PATCH 01/19] feat(connectors): scaffold opensearch_source connector skeleton Mirror elasticsearch_source architecture using the opensearch crate v2.4.0 (rustls-tls) for OpenSearch wire-protocol compatibility. Register as a new workspace member. --- Cargo.lock | 39 ++ Cargo.toml | 2 + .../sources/opensearch_source/Cargo.toml | 51 ++ .../sources/opensearch_source/src/lib.rs | 573 ++++++++++++++++++ .../opensearch_source/src/state_manager.rs | 388 ++++++++++++ 5 files changed, 1053 insertions(+) create mode 100644 core/connectors/sources/opensearch_source/Cargo.toml create mode 100644 core/connectors/sources/opensearch_source/src/lib.rs create mode 100644 core/connectors/sources/opensearch_source/src/state_manager.rs diff --git a/Cargo.lock b/Cargo.lock index d6adc0beb8..ea7b66fdfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6932,6 +6932,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "iggy_connector_opensearch_source" +version = "0.4.1-edge.1" +dependencies = [ + "async-trait", + "dashmap", + "humantime", + "iggy_common", + "iggy_connector_sdk", + "once_cell", + "opensearch", + "secrecy", + "serde", + "serde_json", + "simd-json", + "tokio", + "tracing", +] + [[package]] name = "iggy_connector_postgres_sink" version = "0.4.1-edge.1" @@ -8951,6 +8970,26 @@ dependencies = [ "uuid", ] +[[package]] +name = "opensearch" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af6815a23449a0860c8fe049a828c3589d3ad56d3b5875d0d1f340d1291871e" +dependencies = [ + "base64", + "bytes", + "dyn-clone", + "lazy_static", + "percent-encoding", + "reqwest 0.13.4", + "rustc_version", + "serde", + "serde_json", + "serde_with", + "url", + "void", +] + [[package]] name = "openssl-probe" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 0f9ffd242a..41076fe22f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ members = [ "core/connectors/sinks/stdout_sink", "core/connectors/sources/elasticsearch_source", "core/connectors/sources/influxdb_source", + "core/connectors/sources/opensearch_source", "core/connectors/sources/postgres_source", "core/connectors/sources/random_source", "core/consensus", @@ -206,6 +207,7 @@ nonzero_lit = "0.1.2" notify = "8.2.0" octocrab = "0.51.0" once_cell = "1.21.4" +opensearch = { version = "2.4.0", features = ["rustls-tls"], default-features = false } opentelemetry = { version = "0.32.0", features = ["trace", "logs"] } opentelemetry-appender-tracing = { version = "0.32.0", features = ["log"] } opentelemetry-otlp = { version = "0.32.0", features = [ diff --git a/core/connectors/sources/opensearch_source/Cargo.toml b/core/connectors/sources/opensearch_source/Cargo.toml new file mode 100644 index 0000000000..52eec99e7b --- /dev/null +++ b/core/connectors/sources/opensearch_source/Cargo.toml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "iggy_connector_opensearch_source" +version = "0.4.1-edge.1" +description = "Iggy OpenSearch source connector" +edition = "2024" +license = "Apache-2.0" +keywords = ["iggy", "messaging", "streaming", "opensearch"] +categories = ["command-line-utilities", "database", "network-programming"] +homepage = "https://iggy.apache.org" +documentation = "https://iggy.apache.org/docs" +repository = "https://github.com/apache/iggy" +readme = "../../README.md" +publish = false + +[package.metadata.cargo-machete] +ignored = ["dashmap", "once_cell", "futures", "simd-json"] + +[lib] +crate-type = ["cdylib", "lib"] + +[dependencies] +async-trait = { workspace = true } +dashmap = { workspace = true } +opensearch = { workspace = true } +humantime = { workspace = true } +iggy_common = { workspace = true } +iggy_connector_sdk = { workspace = true } +once_cell = { workspace = true } +secrecy = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +simd-json = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs new file mode 100644 index 0000000000..ddb5c439d3 --- /dev/null +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -0,0 +1,573 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use async_trait::async_trait; +use iggy_common::{DateTime, Utc}; +use iggy_connector_sdk::{ + ConnectorState, Error, ProducedMessage, ProducedMessages, Schema, Source, source_connector, +}; +use opensearch::{ + OpenSearch, SearchParts, + auth::Credentials, + http::{Url, transport::TransportBuilder}, +}; +use secrecy::{ExposeSecret, SecretString}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::str::FromStr; +use std::sync::Arc; +use std::time::Duration; +use tokio::{sync::Mutex, time::sleep}; +use tracing::{info, warn}; + +mod state_manager; +use crate::state_manager::{FileStateStorage, SourceState, StateStorage}; +pub use state_manager::{StateInfo, StateManager, StateStats}; + +source_connector!(OpenSearchSource); + +const CONNECTOR_NAME: &str = "OpenSearch source"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct State { + last_poll_timestamp: Option>, + total_documents_fetched: usize, + poll_count: usize, + /// Last document ID processed (for cursor-based pagination) + last_document_id: Option, + /// Last scroll ID (for scroll-based pagination) + last_scroll_id: Option, + /// Last processed offset + last_offset: Option, + /// Error count and last error + error_count: usize, + last_error: Option, + /// Processing statistics + processing_stats: ProcessingStats, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct ProcessingStats { + /// Total bytes processed + total_bytes_processed: u64, + /// Average processing time per batch + avg_batch_processing_time_ms: f64, + /// Last successful processing timestamp + last_successful_poll: Option>, + /// Number of empty polls + empty_polls_count: usize, + /// Number of successful polls + successful_polls_count: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StateConfig { + /// Enable state persistence + pub enabled: bool, + /// State storage type: "file", "opensearch", "redis", etc. + pub storage_type: Option, + /// State storage configuration (depends on storage_type) + pub storage_config: Option, + /// State ID for this connector instance + pub state_id: Option, + /// Auto-save state interval (e.g., "30s", "5m") + pub auto_save_interval: Option, + /// Fields to track in state (e.g., ["last_timestamp", "last_document_id"]) + pub tracked_fields: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OpenSearchSourceConfig { + pub url: String, + pub index: String, + pub username: Option, + #[serde(serialize_with = "iggy_common::serde_secret::serialize_optional_secret")] + pub password: Option, + pub query: Option, + pub polling_interval: Option, + pub batch_size: Option, + pub timestamp_field: Option, + pub scroll_timeout: Option, + pub state: Option, +} + +#[derive(Debug)] +pub struct OpenSearchSource { + id: u32, + config: OpenSearchSourceConfig, + client: Option, + polling_interval: Duration, + state: Mutex, +} + +impl OpenSearchSource { + pub fn new(id: u32, config: OpenSearchSourceConfig, state: Option) -> Self { + let polling_interval = config + .polling_interval + .as_deref() + .unwrap_or("10s") + .parse::() + .unwrap_or_else(|_| humantime::Duration::from_str("10s").unwrap()) + .into(); + + let restored_state = state + .and_then(|s| s.deserialize::(CONNECTOR_NAME, id)) + .inspect(|s| { + info!( + "Restored state for {CONNECTOR_NAME} connector with ID: {id}. \ + Documents fetched: {}, poll count: {}", + s.total_documents_fetched, s.poll_count + ); + }); + + OpenSearchSource { + id, + config, + client: None, + polling_interval, + state: Mutex::new(restored_state.unwrap_or(State { + last_poll_timestamp: None, + total_documents_fetched: 0, + poll_count: 0, + last_document_id: None, + last_scroll_id: None, + last_offset: None, + error_count: 0, + last_error: None, + processing_stats: ProcessingStats { + total_bytes_processed: 0, + avg_batch_processing_time_ms: 0.0, + last_successful_poll: None, + empty_polls_count: 0, + successful_polls_count: 0, + }, + })), + } + } + + fn serialize_state(&self, state: &State) -> Option { + ConnectorState::serialize(state, CONNECTOR_NAME, self.id) + } + + /// Create state storage based on configuration + fn create_state_storage(&self) -> Option> { + let state_config = self.config.state.as_ref()?; + if !state_config.enabled { + return None; + } + + match state_config.storage_type.as_deref() { + Some("file") | None => { + let base_path = state_config + .storage_config + .as_ref() + .and_then(|c| c.get("base_path")) + .and_then(|p| p.as_str()) + .unwrap_or("./connector_states"); + + Some(Arc::new(FileStateStorage::new(base_path))) + } + Some("opensearch") => { + // TODO: Implement OpenSearch-based state storage + warn!("OpenSearch state storage not yet implemented, falling back to file storage"); + Some(Arc::new(FileStateStorage::new("./connector_states"))) + } + Some(storage_type) => { + warn!( + "Unknown state storage type: {}, falling back to file storage", + storage_type + ); + Some(Arc::new(FileStateStorage::new("./connector_states"))) + } + } + } + + /// Get state ID for this connector + fn get_state_id(&self) -> String { + self.config + .state + .as_ref() + .and_then(|s| s.state_id.clone()) + .unwrap_or_else(|| format!("opensearch_source_{}", self.id)) + } + + /// Convert internal state to SourceState + async fn internal_state_to_source_state(&self) -> Result { + let state = self.state.lock().await; + + let data = json!({ + "last_poll_timestamp": state.last_poll_timestamp, + "total_documents_fetched": state.total_documents_fetched, + "poll_count": state.poll_count, + "last_document_id": state.last_document_id, + "last_scroll_id": state.last_scroll_id, + "last_offset": state.last_offset, + "error_count": state.error_count, + "last_error": state.last_error, + "processing_stats": state.processing_stats, + }); + + Ok(SourceState { + id: self.get_state_id(), + last_updated: Utc::now(), + version: 1, + data, + metadata: Some(json!({ + "connector_type": "opensearch_source", + "connector_id": self.id, + "index": self.config.index, + "url": self.config.url, + })), + }) + } + + /// Convert SourceState to internal state + async fn source_state_to_internal_state( + &mut self, + source_state: SourceState, + ) -> Result<(), Error> { + let mut state = self.state.lock().await; + + if let Some(data) = source_state.data.as_object() { + if let Some(timestamp) = data.get("last_poll_timestamp") + && let Some(ts_str) = timestamp.as_str() + && let Ok(dt) = DateTime::parse_from_rfc3339(ts_str) + { + state.last_poll_timestamp = Some(dt.with_timezone(&Utc)); + } + + if let Some(count) = data.get("total_documents_fetched") + && let Some(count_val) = count.as_u64() + { + state.total_documents_fetched = count_val as usize; + } + + if let Some(count) = data.get("poll_count") + && let Some(count_val) = count.as_u64() + { + state.poll_count = count_val as usize; + } + + if let Some(doc_id) = data.get("last_document_id") { + state.last_document_id = doc_id.as_str().map(|s| s.to_string()); + } + + if let Some(scroll_id) = data.get("last_scroll_id") { + state.last_scroll_id = scroll_id.as_str().map(|s| s.to_string()); + } + + if let Some(offset) = data.get("last_offset") { + state.last_offset = offset.as_u64(); + } + + if let Some(error_count) = data.get("error_count") + && let Some(count_val) = error_count.as_u64() + { + state.error_count = count_val as usize; + } + + if let Some(last_error) = data.get("last_error") { + state.last_error = last_error.as_str().map(|s| s.to_string()); + } + + if let Some(stats) = data.get("processing_stats") + && let Ok(processing_stats) = serde_json::from_value(stats.clone()) + { + state.processing_stats = processing_stats; + } + } + + Ok(()) + } + + async fn create_client(&self) -> Result { + let url = Url::parse(&self.config.url) + .map_err(|error| Error::Storage(format!("Invalid OpenSearch URL: {error}")))?; + + let conn_pool = opensearch::http::transport::SingleNodeConnectionPool::new(url); + let mut transport_builder = TransportBuilder::new(conn_pool); + + if let (Some(username), Some(password)) = (&self.config.username, &self.config.password) { + let credentials = + Credentials::Basic(username.clone(), password.expose_secret().to_string()); + transport_builder = transport_builder.auth(credentials); + } + + let transport = transport_builder + .build() + .map_err(|e| Error::Storage(format!("Failed to build transport: {}", e)))?; + + Ok(OpenSearch::new(transport)) + } + + async fn search_documents(&self, client: &OpenSearch) -> Result, Error> { + let state = self.state.lock().await; + let batch_size = self.config.batch_size.unwrap_or(100); + + // Build query based on timestamp field if configured + let mut query = self.config.query.clone().unwrap_or_else(|| { + json!({ + "match_all": {} + }) + }); + + // Add timestamp filter for incremental polling + if let Some(timestamp_field) = &self.config.timestamp_field + && let Some(last_timestamp) = state.last_poll_timestamp + { + query = json!({ + "bool": { + "must": [ + query, + { + "range": { + timestamp_field: { + "gt": last_timestamp.to_rfc3339() + } + } + } + ] + } + }); + } + + let search_body = json!({ + "query": query, + "size": batch_size, + "sort": [ + { + self.config.timestamp_field.as_deref().unwrap_or("@timestamp"): { + "order": "asc" + } + } + ] + }); + + drop(state); + + let response = client + .search(SearchParts::Index(&[&self.config.index])) + .body(search_body) + .send() + .await + .map_err(|e| Error::Storage(format!("Failed to execute search: {}", e)))?; + + if !response.status_code().is_success() { + let error_text = response + .text() + .await + .unwrap_or_else(|_| "Unknown error".to_string()); + return Err(Error::Storage(format!( + "Search request failed: {}", + error_text + ))); + } + + let response_body: Value = response + .json() + .await + .map_err(|e| Error::Storage(format!("Failed to parse search response: {}", e)))?; + + let mut messages = Vec::new(); + let mut latest_timestamp = None; + + if let Some(hits) = response_body + .get("hits") + .and_then(|h| h.get("hits")) + .and_then(|h| h.as_array()) + { + for hit in hits { + if let Some(source) = hit.get("_source") { + // Extract timestamp for incremental polling + if let Some(timestamp_field) = &self.config.timestamp_field + && let Some(timestamp_str) = + source.get(timestamp_field).and_then(|v| v.as_str()) + && let Ok(timestamp) = DateTime::parse_from_rfc3339(timestamp_str) + { + let timestamp_utc = timestamp.with_timezone(&Utc); + if latest_timestamp.is_none() || timestamp_utc > latest_timestamp.unwrap() { + latest_timestamp = Some(timestamp_utc); + } + } + + // Create message from document + let payload = serde_json::to_vec(source).map_err(|e| { + Error::Serialization(format!("Failed to serialize document: {}", e)) + })?; + + let message = ProducedMessage { + id: None, + headers: None, + checksum: None, + timestamp: None, + origin_timestamp: None, + payload, + }; + messages.push(message); + } + } + } + + // Update state + let mut state = self.state.lock().await; + state.total_documents_fetched += messages.len(); + state.poll_count += 1; + if let Some(timestamp) = latest_timestamp { + state.last_poll_timestamp = Some(timestamp); + } + + Ok(messages) + } +} + +#[async_trait] +impl Source for OpenSearchSource { + async fn open(&mut self) -> Result<(), Error> { + info!( + "Opening OpenSearch source connector with ID: {} for URL: {}, index: {}", + self.id, self.config.url, self.config.index + ); + + let client = self.create_client().await?; + + // Test connection by checking if index exists + let response = client + .indices() + .exists(opensearch::indices::IndicesExistsParts::Index(&[&self + .config + .index])) + .send() + .await + .map_err(|e| Error::Storage(format!("Failed to check index existence: {}", e)))?; + + if !response.status_code().is_success() { + return Err(Error::Storage(format!( + "Index '{}' does not exist or is not accessible", + self.config.index + ))); + } + + self.client = Some(client); + + // Load state if state management is enabled + if self + .config + .state + .as_ref() + .map(|s| s.enabled) + .unwrap_or(false) + && let Err(e) = self.load_state().await + { + warn!( + "Failed to load state for OpenSearch source connector with ID: {}: {}", + self.id, e + ); + } + + info!( + "Successfully opened OpenSearch source connector with ID: {}", + self.id + ); + Ok(()) + } + + async fn poll(&self) -> Result { + let start_time = std::time::Instant::now(); + + sleep(self.polling_interval).await; + + let client = self + .client + .as_ref() + .ok_or_else(|| Error::Storage("OpenSearch client not initialized".to_string()))?; + + let messages = match self.search_documents(client).await { + Ok(msgs) => { + // Update success statistics + let mut state = self.state.lock().await; + state.processing_stats.successful_polls_count += 1; + state.processing_stats.last_successful_poll = Some(Utc::now()); + + let processing_time = start_time.elapsed().as_millis() as f64; + let total_polls = state.processing_stats.successful_polls_count + + state.processing_stats.empty_polls_count; + state.processing_stats.avg_batch_processing_time_ms = + (state.processing_stats.avg_batch_processing_time_ms + * (total_polls - 1) as f64 + + processing_time) + / total_polls as f64; + + if msgs.is_empty() { + state.processing_stats.empty_polls_count += 1; + } + + drop(state); + msgs + } + Err(e) => { + // Update error statistics + let mut state = self.state.lock().await; + state.error_count += 1; + state.last_error = Some(e.to_string()); + drop(state); + return Err(e); + } + }; + let persisted_state = { + let state = self.state.lock().await; + self.serialize_state(&state) + }; + + Ok(ProducedMessages { + schema: Schema::Json, + messages, + state: persisted_state, + }) + } + + async fn close(&mut self) -> Result<(), Error> { + let state = self.state.lock().await; + info!( + "OpenSearch source connector with ID: {} is closing. Stats: {} total documents fetched, {} polls executed, {} errors", + self.id, state.total_documents_fetched, state.poll_count, state.error_count + ); + drop(state); + + // Save final state if state management is enabled + if self + .config + .state + .as_ref() + .map(|s| s.enabled) + .unwrap_or(false) + && let Err(e) = self.save_state().await + { + warn!( + "Failed to save final state for OpenSearch source connector with ID: {}: {}", + self.id, e + ); + } + + self.client = None; + info!( + "OpenSearch source connector with ID: {} is closed.", + self.id + ); + Ok(()) + } +} diff --git a/core/connectors/sources/opensearch_source/src/state_manager.rs b/core/connectors/sources/opensearch_source/src/state_manager.rs new file mode 100644 index 0000000000..c681444efb --- /dev/null +++ b/core/connectors/sources/opensearch_source/src/state_manager.rs @@ -0,0 +1,388 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use crate::{OpenSearchSource, StateConfig}; +use async_trait::async_trait; +use iggy_common::{ChronoDuration, DateTime, Utc}; +use iggy_connector_sdk::Error; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; +use std::sync::Arc; +use tokio::time::{Duration, interval}; +use tracing::{error, info, warn}; + +impl OpenSearchSource { + async fn get_state(&self) -> Result, Error> { + if self + .config + .state + .as_ref() + .map(|s| s.enabled) + .unwrap_or(false) + { + Ok(Some(self.internal_state_to_source_state().await?)) + } else { + Ok(None) + } + } + + pub(super) async fn save_state(&self) -> Result<(), Error> { + if !self + .config + .state + .as_ref() + .map(|s| s.enabled) + .unwrap_or(false) + { + return Ok(()); + } + + let storage = self + .create_state_storage() + .ok_or_else(|| Error::Storage("State storage not configured".to_string()))?; + + let source_state = self.internal_state_to_source_state().await?; + storage.save_source_state(&source_state).await?; + + info!( + "Saved state for OpenSearch source connector with ID: {}", + self.id + ); + Ok(()) + } + + pub(super) async fn load_state(&mut self) -> Result<(), Error> { + if !self + .config + .state + .as_ref() + .map(|s| s.enabled) + .unwrap_or(false) + { + return Ok(()); + } + + let storage = self + .create_state_storage() + .ok_or_else(|| Error::Storage("State storage not configured".to_string()))?; + + let state_id = self.get_state_id(); + if let Some(source_state) = storage.load_source_state(&state_id).await? { + self.source_state_to_internal_state(source_state).await?; + + let state = self.state.lock().await; + info!( + "Loaded state for OpenSearch source connector with ID: {} - last poll: {:?}, total docs: {}, polls: {}", + self.id, state.last_poll_timestamp, state.total_documents_fetched, state.poll_count + ); + } else { + info!( + "No existing state found for OpenSearch source connector with ID: {}, starting fresh", + self.id + ); + } + + Ok(()) + } +} + +/// State manager for OpenSearch source connector +pub struct StateManager { + storage: Arc, + config: StateConfig, + auto_save_interval: Option, +} + +impl StateManager { + pub fn new(config: StateConfig) -> Result { + let storage = Self::create_storage(&config)?; + let auto_save_interval = config + .auto_save_interval + .as_deref() + .and_then(|interval_str| { + humantime::Duration::from_str(interval_str) + .ok() + .map(|d| Duration::from_secs(d.as_secs())) + }); + + Ok(Self { + storage, + config, + auto_save_interval, + }) + } + + fn create_storage(config: &StateConfig) -> Result, Error> { + match config.storage_type.as_deref() { + Some("file") | None => { + let base_path = config + .storage_config + .as_ref() + .and_then(|c| c.get("base_path")) + .and_then(|p| p.as_str()) + .unwrap_or("./connector_states"); + + Ok(Arc::new(FileStateStorage::new(base_path))) + } + Some("opensearch") => { + // TODO: Implement OpenSearch-based state storage + warn!("OpenSearch state storage not yet implemented, falling back to file storage"); + Ok(Arc::new(FileStateStorage::new("./connector_states"))) + } + Some(storage_type) => { + warn!( + "Unknown state storage type: {}, falling back to file storage", + storage_type + ); + Ok(Arc::new(FileStateStorage::new("./connector_states"))) + } + } + } + + /// Start auto-save background task + pub async fn start_auto_save(&self, connector: Arc) { + let interval_duration = self + .auto_save_interval + .unwrap_or_else(|| Duration::from_secs(60)); + let storage = self.storage.clone(); + let state_id = self.config.state_id.clone(); + tokio::spawn(async move { + let mut interval = interval(interval_duration); + loop { + interval.tick().await; + if let Ok(Some(state)) = connector.get_state().await { + if let Err(e) = storage.save_source_state(&state).await { + error!( + "Failed to auto-save state for {}: {}", + state_id.as_deref().unwrap_or("unknown"), + e + ); + } else { + info!( + "Auto-saved state for {}", + state_id.as_deref().unwrap_or("unknown") + ); + } + } + } + }); + } + + /// Get state statistics + pub async fn get_state_stats(&self) -> Result { + let state_ids = self.storage.list_states().await?; + let mut stats = StateStats { + total_states: state_ids.len(), + states: Vec::new(), + }; + + for state_id in state_ids { + if let Some(state) = self.storage.load_source_state(&state_id).await? { + stats.states.push(StateInfo { + id: state.id, + last_updated: state.last_updated, + version: state.version, + connector_type: state + .metadata + .as_ref() + .and_then(|m| m.get("connector_type")) + .and_then(|t| t.as_str()) + .unwrap_or("unknown") + .to_string(), + }); + } + } + + Ok(stats) + } + + /// Clean up old states + pub async fn cleanup_old_states(&self, older_than_days: u32) -> Result { + let state_ids = self.storage.list_states().await?; + let cutoff_time = Utc::now() - ChronoDuration::days(older_than_days as i64); + let mut deleted_count = 0; + + for state_id in state_ids { + if let Some(state) = self.storage.load_source_state(&state_id).await? + && state.last_updated < cutoff_time + { + if let Err(e) = self.storage.delete_state(&state_id).await { + warn!("Failed to delete old state {}: {}", state_id, e); + } else { + deleted_count += 1; + info!("Deleted old state: {}", state_id); + } + } + } + + Ok(deleted_count) + } + + pub fn auto_save_interval(&self) -> Option { + self.auto_save_interval + } +} + +#[derive(Debug)] +pub struct StateStats { + pub total_states: usize, + pub states: Vec, +} + +#[derive(Debug)] +pub struct StateInfo { + pub id: String, + pub last_updated: DateTime, + pub version: u32, + pub connector_type: String, +} + +/// State management for source connectors +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SourceState { + /// Unique identifier for this state + pub id: String, + /// Timestamp when this state was last updated + pub last_updated: DateTime, + /// Version of the state format + pub version: u32, + /// Generic state data as JSON + pub data: serde_json::Value, + /// Optional metadata + pub metadata: Option, +} + +/// State storage backend trait +#[async_trait] +pub trait StateStorage: Send + Sync { + /// Save source state to storage + async fn save_source_state(&self, state: &SourceState) -> Result<(), Error>; + + /// Load source state from storage + async fn load_source_state(&self, id: &str) -> Result, Error>; + + /// Delete state from storage + async fn delete_state(&self, id: &str) -> Result<(), Error>; + + /// List all state IDs + async fn list_states(&self) -> Result, Error>; +} + +/// File-based state storage implementation +pub struct FileStateStorage { + base_path: std::path::PathBuf, +} + +impl FileStateStorage { + pub fn new>(base_path: P) -> Self { + Self { + base_path: base_path.as_ref().to_path_buf(), + } + } + + fn get_state_path(&self, id: &str) -> std::path::PathBuf { + self.base_path.join(format!("{id}.json")) + } +} + +#[async_trait] +impl StateStorage for FileStateStorage { + async fn save_source_state(&self, state: &SourceState) -> Result<(), Error> { + use tokio::fs; + + // Ensure directory exists + if let Some(parent) = self.base_path.parent() { + fs::create_dir_all(parent) + .await + .map_err(|e| Error::Storage(format!("Failed to create state directory: {e}")))?; + } + + let path = self.get_state_path(&state.id); + let json = serde_json::to_string_pretty(state) + .map_err(|e| Error::Serialization(format!("Failed to serialize source state: {e}")))?; + + fs::write(path, json) + .await + .map_err(|e| Error::Storage(format!("Failed to write state file: {e}")))?; + + Ok(()) + } + + async fn load_source_state(&self, id: &str) -> Result, Error> { + use tokio::fs; + + let path = self.get_state_path(id); + if !path.exists() { + return Ok(None); + } + + let content = fs::read_to_string(path) + .await + .map_err(|e| Error::Storage(format!("Failed to read state file: {e}")))?; + + let state: SourceState = serde_json::from_str(&content).map_err(|e| { + Error::Serialization(format!("Failed to deserialize source state: {e}")) + })?; + + Ok(Some(state)) + } + + async fn delete_state(&self, id: &str) -> Result<(), Error> { + use tokio::fs; + + let path = self.get_state_path(id); + if path.exists() { + fs::remove_file(path) + .await + .map_err(|e| Error::Storage(format!("Failed to delete state file: {e}")))?; + } + + Ok(()) + } + + async fn list_states(&self) -> Result, Error> { + use tokio::fs; + + let mut states = Vec::new(); + + if !self.base_path.exists() { + return Ok(states); + } + + let mut entries = fs::read_dir(&self.base_path) + .await + .map_err(|e| Error::Storage(format!("Failed to read state directory: {e}")))?; + + while let Some(entry) = entries + .next_entry() + .await + .map_err(|e| Error::Storage(format!("Failed to read directory entry: {e}")))? + { + if let Some(extension) = entry.path().extension() + && extension == "json" + && let Some(stem) = entry.path().file_stem() + && let Some(id) = stem.to_str() + { + states.push(id.to_string()); + } + } + + Ok(states) + } +} From 143b26c98feddd1cc38a6ae8dde9ff13f56316df Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 14 Jun 2026 00:21:26 -0400 Subject: [PATCH 02/19] docs(connectors): add opensearch_source config and README Mirror elasticsearch_source documentation, adapted for the OpenSearch wire protocol and state storage type naming. --- .../sources/opensearch_source/README.md | 211 ++++++++++++++++++ .../sources/opensearch_source/config.toml | 38 ++++ 2 files changed, 249 insertions(+) create mode 100644 core/connectors/sources/opensearch_source/README.md create mode 100644 core/connectors/sources/opensearch_source/config.toml diff --git a/core/connectors/sources/opensearch_source/README.md b/core/connectors/sources/opensearch_source/README.md new file mode 100644 index 0000000000..466b20f7b4 --- /dev/null +++ b/core/connectors/sources/opensearch_source/README.md @@ -0,0 +1,211 @@ +# OpenSearch Source Connector with State Management + +This OpenSearch source connector provides comprehensive state management capabilities to track processing progress and enable fault-tolerant data ingestion. + +## Features + +- **Incremental Data Processing**: Track last processed timestamp to avoid reprocessing data +- **Cursor-based Pagination**: Support for document ID-based cursors +- **Scroll-based Pagination**: Support for OpenSearch scroll API +- **Error Tracking**: Monitor error counts and last error messages +- **Processing Statistics**: Track performance metrics and processing times +- **Persistent State Storage**: Multiple storage backends (file, OpenSearch, Redis) +- **Auto-save**: Configurable automatic state persistence +- **State Recovery**: Resume processing from last known position after restart + +## Configuration + +### Basic Configuration + +```toml +type = "source" +key = "opensearch" +enabled = true +version = 0 +name = "OpenSearch source" +path = "target/release/libiggy_connector_opensearch_source" + +[[streams]] +stream = "opensearch_stream" +topic = "documents" +schema = "json" +batch_length = 100 +linger_time = "5ms" + +[plugin_config] +url = "http://localhost:9200" +index = "logs-*" +polling_interval = "30s" +batch_size = 100 +timestamp_field = "@timestamp" +query = { + "match_all": {} +} +``` + +### State Management Configuration + +```toml +[plugin_config] +# ... basic config ... +state = { + enabled = true + storage_type = "file" # "file", "opensearch", "redis" + storage_config = { + base_path = "./connector_states" # for file storage + # index = "connector_states" # for opensearch storage + # url = "redis://localhost:6379" # for redis storage + } + state_id = "opensearch_logs_connector" + auto_save_interval = "5m" + tracked_fields = [ + "last_poll_timestamp", + "last_document_id", + "total_documents_fetched" + ] +} +``` + +## State Information + +The connector tracks the following state information: + +### Processing State + +- `last_poll_timestamp`: Last successful poll timestamp +- `total_documents_fetched`: Total number of documents processed +- `poll_count`: Number of polling cycles executed +- `last_document_id`: Last processed document ID (for cursor pagination) +- `last_scroll_id`: Last scroll ID (for scroll pagination) +- `last_offset`: Last processed offset + +### Error Tracking + +- `error_count`: Total number of errors encountered +- `last_error`: Last error message + +### Performance Statistics + +- `total_bytes_processed`: Total bytes processed +- `avg_batch_processing_time_ms`: Average processing time per batch +- `last_successful_poll`: Timestamp of last successful poll +- `empty_polls_count`: Number of polls that returned no documents +- `successful_polls_count`: Number of successful polls + +## Storage Backends + +### File Storage (Default) + +```toml +state = { + enabled = true + storage_type = "file" + storage_config = { + base_path = "./connector_states" + } +} +``` + +### OpenSearch Storage + +```toml +state = { + enabled = true + storage_type = "opensearch" + storage_config = { + index = "connector_states" + url = "http://localhost:9200" + } +} +``` + +### Redis Storage + +```toml +state = { + enabled = true + storage_type = "redis" + storage_config = { + url = "redis://localhost:6379" + key_prefix = "connector_states:" + } +} +``` + +## State File Format + +State files are stored as JSON with the following structure: + +```json +{ + "id": "opensearch_logs_connector", + "last_updated": "2024-01-15T10:30:00Z", + "version": 1, + "data": { + "last_poll_timestamp": "2024-01-15T10:30:00Z", + "total_documents_fetched": 15000, + "poll_count": 150, + "last_document_id": "doc_12345", + "last_scroll_id": "scroll_abc123", + "last_offset": 15000, + "error_count": 2, + "last_error": "Connection timeout", + "processing_stats": { + "total_bytes_processed": 1048576, + "avg_batch_processing_time_ms": 125.5, + "last_successful_poll": "2024-01-15T10:30:00Z", + "empty_polls_count": 5, + "successful_polls_count": 145 + } + }, + "metadata": { + "connector_type": "opensearch_source", + "connector_id": 1, + "index": "logs-*", + "url": "http://localhost:9200" + } +} +``` + +## Best Practices + +1. **State ID Uniqueness**: Use unique state IDs for different connector instances +2. **Auto-save Interval**: Set appropriate auto-save intervals based on your data volume +3. **Storage Location**: Use persistent storage locations for production deployments +4. **State Cleanup**: Regularly clean up old state files to prevent disk space issues +5. **Error Handling**: Monitor error counts and implement appropriate alerting +6. **Backup**: Regularly backup state files for disaster recovery + +## Troubleshooting + +### Common Issues + +1. **State Not Loading**: Check file permissions and storage path +2. **State Corruption**: Delete corrupted state files to start fresh +3. **Performance Issues**: Adjust auto-save interval and batch sizes +4. **Storage Full**: Implement state cleanup policies + +### Monitoring + +Monitor the following metrics: + +- State save/load success rates +- Processing statistics +- Error counts and types +- Storage usage for state files + +## Migration + +To migrate from a connector without state management: + +1. Add state configuration to your connector config +2. Set `enabled = true` in state config +3. Restart the connector +4. The connector will start tracking state from the next poll cycle + +To migrate between storage backends: + +1. Export state from current storage +2. Update storage configuration +3. Import state to new storage +4. Restart connector diff --git a/core/connectors/sources/opensearch_source/config.toml b/core/connectors/sources/opensearch_source/config.toml new file mode 100644 index 0000000000..dceed1b064 --- /dev/null +++ b/core/connectors/sources/opensearch_source/config.toml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "source" +key = "opensearch" +enabled = true +version = 0 +name = "OpenSearch source" +path = "../../target/release/libiggy_connector_opensearch_source" +plugin_config_format = "json" + +[[streams]] +stream = "test_stream" +topic = "test_topic" +schema = "json" +batch_length = 1000 +linger_time = "5ms" + +[plugin_config] +url = "http://localhost:9200" +index = "test_documents" +polling_interval = "100ms" +batch_size = 100 +timestamp_field = "timestamp" From 4367a452299040bfb03ae3ac76d22638affd93ce Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 14 Jun 2026 00:29:18 -0400 Subject: [PATCH 03/19] test(connectors): add opensearch_source unit and integration tests Add the four canonical source-state unit tests, plus integration fixtures (opensearchproject/opensearch:2.19.1 container) and end-to-end tests covering happy paths (poll, empty index, bulk, restart state persistence) and a negative path (missing index surfaces ConnectorStatus::Error via the runtime API). --- Cargo.lock | 1 + .../sources/opensearch_source/Cargo.toml | 3 +- .../sources/opensearch_source/src/lib.rs | 115 ++++++ .../tests/connectors/fixtures/mod.rs | 2 + .../fixtures/opensearch/container.rs | 349 +++++++++++++++++ .../connectors/fixtures/opensearch/mod.rs | 23 ++ .../connectors/fixtures/opensearch/source.rs | 195 ++++++++++ core/integration/tests/connectors/mod.rs | 1 + .../tests/connectors/opensearch/mod.rs | 24 ++ .../opensearch/opensearch_source.rs | 367 ++++++++++++++++++ .../tests/connectors/opensearch/source.toml | 20 + 11 files changed, 1099 insertions(+), 1 deletion(-) create mode 100644 core/integration/tests/connectors/fixtures/opensearch/container.rs create mode 100644 core/integration/tests/connectors/fixtures/opensearch/mod.rs create mode 100644 core/integration/tests/connectors/fixtures/opensearch/source.rs create mode 100644 core/integration/tests/connectors/opensearch/mod.rs create mode 100644 core/integration/tests/connectors/opensearch/opensearch_source.rs create mode 100644 core/integration/tests/connectors/opensearch/source.toml diff --git a/Cargo.lock b/Cargo.lock index ea7b66fdfd..f3fec351ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6943,6 +6943,7 @@ dependencies = [ "iggy_connector_sdk", "once_cell", "opensearch", + "rmp-serde", "secrecy", "serde", "serde_json", diff --git a/core/connectors/sources/opensearch_source/Cargo.toml b/core/connectors/sources/opensearch_source/Cargo.toml index 52eec99e7b..5c91507112 100644 --- a/core/connectors/sources/opensearch_source/Cargo.toml +++ b/core/connectors/sources/opensearch_source/Cargo.toml @@ -38,11 +38,12 @@ crate-type = ["cdylib", "lib"] [dependencies] async-trait = { workspace = true } dashmap = { workspace = true } -opensearch = { workspace = true } humantime = { workspace = true } iggy_common = { workspace = true } iggy_connector_sdk = { workspace = true } once_cell = { workspace = true } +opensearch = { workspace = true } +rmp-serde = { workspace = true } secrecy = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs index ddb5c439d3..498d0bf386 100644 --- a/core/connectors/sources/opensearch_source/src/lib.rs +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -571,3 +571,118 @@ impl Source for OpenSearchSource { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn test_config() -> OpenSearchSourceConfig { + OpenSearchSourceConfig { + url: "http://localhost:9200".to_string(), + index: "test_documents".to_string(), + username: None, + password: None, + query: None, + polling_interval: Some("100ms".to_string()), + batch_size: Some(10), + timestamp_field: Some("timestamp".to_string()), + scroll_timeout: None, + state: None, + } + } + + fn test_state() -> State { + State { + last_poll_timestamp: None, + total_documents_fetched: 500, + poll_count: 5, + last_document_id: Some("doc_42".to_string()), + last_scroll_id: None, + last_offset: Some(500), + error_count: 1, + last_error: Some("connection reset".to_string()), + processing_stats: ProcessingStats { + total_bytes_processed: 1024, + avg_batch_processing_time_ms: 12.5, + last_successful_poll: None, + empty_polls_count: 2, + successful_polls_count: 5, + }, + } + } + + #[test] + fn given_persisted_state_should_restore_total_documents_fetched() { + let state = test_state(); + let serialized = rmp_serde::to_vec(&state).expect("Failed to serialize state"); + let connector_state = ConnectorState(serialized); + + let source = OpenSearchSource::new(1, test_config(), Some(connector_state)); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let restored = source.state.lock().await; + assert_eq!(restored.total_documents_fetched, 500); + assert_eq!(restored.poll_count, 5); + assert_eq!(restored.last_document_id, Some("doc_42".to_string())); + }); + } + + #[test] + fn given_no_state_should_start_fresh() { + let source = OpenSearchSource::new(1, test_config(), None); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let state = source.state.lock().await; + assert_eq!(state.total_documents_fetched, 0); + assert_eq!(state.poll_count, 0); + assert_eq!(state.last_document_id, None); + }); + } + + #[test] + fn given_invalid_state_should_start_fresh() { + let invalid_state = ConnectorState(b"not valid msgpack".to_vec()); + let source = OpenSearchSource::new(1, test_config(), Some(invalid_state)); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let state = source.state.lock().await; + assert_eq!(state.total_documents_fetched, 0); + assert_eq!(state.poll_count, 0); + }); + } + + #[test] + fn state_should_be_serializable_and_deserializable() { + let original = test_state(); + + let serialized = rmp_serde::to_vec(&original).expect("Failed to serialize"); + let deserialized: State = + rmp_serde::from_slice(&serialized).expect("Failed to deserialize"); + + assert_eq!( + original.total_documents_fetched, + deserialized.total_documents_fetched + ); + assert_eq!(original.poll_count, deserialized.poll_count); + assert_eq!(original.last_document_id, deserialized.last_document_id); + assert_eq!(original.error_count, deserialized.error_count); + } + + #[test] + fn serialize_state_helper_should_produce_valid_connector_state() { + let source = OpenSearchSource::new(1, test_config(), None); + let state = test_state(); + + let connector_state = source.serialize_state(&state); + assert!(connector_state.is_some()); + + let restored: State = connector_state + .unwrap() + .deserialize(CONNECTOR_NAME, 1) + .expect("Failed to deserialize state"); + assert_eq!(restored.total_documents_fetched, 500); + } +} diff --git a/core/integration/tests/connectors/fixtures/mod.rs b/core/integration/tests/connectors/fixtures/mod.rs index 616c3a557a..4fac6db286 100644 --- a/core/integration/tests/connectors/fixtures/mod.rs +++ b/core/integration/tests/connectors/fixtures/mod.rs @@ -26,6 +26,7 @@ mod http; mod iceberg; mod influxdb; mod mongodb; +mod opensearch; mod postgres; mod quickwit; mod wiremock; @@ -68,6 +69,7 @@ pub use mongodb::{ MongoDbOps, MongoDbSinkAutoCreateFixture, MongoDbSinkBatchFixture, MongoDbSinkFailpointFixture, MongoDbSinkFixture, MongoDbSinkJsonFixture, MongoDbSinkWriteConcernFixture, }; +pub use opensearch::{OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture}; pub use postgres::{ PostgresOps, PostgresSinkByteaFixture, PostgresSinkFixture, PostgresSinkJsonFixture, PostgresSourceByteaFixture, PostgresSourceDeleteFixture, PostgresSourceJsonFixture, diff --git a/core/integration/tests/connectors/fixtures/opensearch/container.rs b/core/integration/tests/connectors/fixtures/opensearch/container.rs new file mode 100644 index 0000000000..245acab021 --- /dev/null +++ b/core/integration/tests/connectors/fixtures/opensearch/container.rs @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use integration::harness::TestBinaryError; +use reqwest_middleware::ClientWithMiddleware as HttpClient; +use reqwest_retry::RetryTransientMiddleware; +use reqwest_retry::policies::ExponentialBackoff; +use serde::Deserialize; +use testcontainers_modules::testcontainers::core::wait::HttpWaitStrategy; +use testcontainers_modules::testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers_modules::testcontainers::runners::AsyncRunner; +use testcontainers_modules::testcontainers::{ + ContainerAsync, GenericImage, ImageExt, ReuseDirective, +}; +use tracing::info; + +const OPENSEARCH_IMAGE: &str = "docker.io/opensearchproject/opensearch"; +const OPENSEARCH_TAG: &str = "2.19.1"; +const OPENSEARCH_PORT: u16 = 9200; +const OPENSEARCH_HEALTH_ENDPOINT: &str = "/_cluster/health"; +// Fixed name + ReuseDirective::Always shares one container across nextest's +// per-test processes: the first test creates it, every later test attaches by +// name. Per-test isolation comes from a unique index per fixture, not a fresh +// container. +const OPENSEARCH_CONTAINER_NAME: &str = "iggy-test-opensearch"; + +pub const DEFAULT_TEST_STREAM: &str = "test_stream"; +pub const DEFAULT_TEST_TOPIC: &str = "test_topic"; + +pub const ENV_SOURCE_URL: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_URL"; +pub const ENV_SOURCE_INDEX: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_INDEX"; +pub const ENV_SOURCE_POLLING_INTERVAL: &str = + "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_POLLING_INTERVAL"; +pub const ENV_SOURCE_BATCH_SIZE: &str = + "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_BATCH_SIZE"; +pub const ENV_SOURCE_TIMESTAMP_FIELD: &str = + "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_TIMESTAMP_FIELD"; +pub const ENV_SOURCE_STREAMS_0_STREAM: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_STREAMS_0_STREAM"; +pub const ENV_SOURCE_STREAMS_0_TOPIC: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_STREAMS_0_TOPIC"; +pub const ENV_SOURCE_STREAMS_0_SCHEMA: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_STREAMS_0_SCHEMA"; +pub const ENV_SOURCE_PATH: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PATH"; + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +pub struct OpenSearchSearchResponse { + pub hits: OpenSearchHits, +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +pub struct OpenSearchHits { + pub total: OpenSearchTotal, + pub hits: Vec, +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +pub struct OpenSearchTotal { + pub value: usize, +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +pub struct OpenSearchHit { + #[serde(rename = "_source")] + pub source: serde_json::Value, +} + +pub struct OpenSearchContainer { + // Held so testcontainers' Drop runs on test exit; ReuseDirective::Always + // makes that Drop leave the container running for the next test to attach. + #[allow(dead_code)] + container: ContainerAsync, + pub base_url: String, +} + +impl OpenSearchContainer { + pub async fn start() -> Result { + let container = GenericImage::new(OPENSEARCH_IMAGE, OPENSEARCH_TAG) + .with_exposed_port(OPENSEARCH_PORT.tcp()) + .with_wait_for(WaitFor::http( + HttpWaitStrategy::new(OPENSEARCH_HEALTH_ENDPOINT) + .with_port(OPENSEARCH_PORT.tcp()) + .with_expected_status_code(200u16), + )) + .with_startup_timeout(std::time::Duration::from_secs(120)) + .with_env_var("discovery.type", "single-node") + .with_env_var("plugins.security.disabled", "true") + .with_env_var("OPENSEARCH_JAVA_OPTS", "-Xms512m -Xmx512m") + .with_mapped_port(0, OPENSEARCH_PORT.tcp()) + .with_container_name(OPENSEARCH_CONTAINER_NAME) + .with_reuse(ReuseDirective::Always) + .start() + .await + .map_err(|e| TestBinaryError::FixtureSetup { + fixture_type: "OpenSearchContainer".to_string(), + message: format!("Failed to start container: {e}"), + })?; + + info!("Started OpenSearch container"); + + let mapped_port = container + .ports() + .await + .map_err(|e| TestBinaryError::FixtureSetup { + fixture_type: "OpenSearchContainer".to_string(), + message: format!("Failed to get ports: {e}"), + })? + .map_to_host_port_ipv4(OPENSEARCH_PORT) + .ok_or_else(|| TestBinaryError::FixtureSetup { + fixture_type: "OpenSearchContainer".to_string(), + message: "No mapping for OpenSearch port".to_string(), + })?; + + let base_url = format!("http://localhost:{mapped_port}"); + info!("OpenSearch container available at {base_url}"); + + Ok(Self { + container, + base_url, + }) + } +} + +pub fn create_http_client() -> HttpClient { + let retry_policy = ExponentialBackoff::builder().build_with_max_retries(3); + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .expect("Failed to build HTTP client"); + reqwest_middleware::ClientBuilder::new(client) + .with(RetryTransientMiddleware::new_with_policy(retry_policy)) + .build() +} + +pub trait OpenSearchOps: Sync { + fn container(&self) -> &OpenSearchContainer; + fn http_client(&self) -> &HttpClient; + + fn create_index( + &self, + index_name: &str, + ) -> impl std::future::Future> + Send { + async move { + let url = format!("{}/{}", self.container().base_url, index_name); + let mapping = serde_json::json!({ + "mappings": { + "properties": { + "id": { "type": "integer" }, + "name": { "type": "keyword" }, + "value": { "type": "integer" }, + "timestamp": { "type": "date" } + } + } + }); + + let response = self + .http_client() + .put(&url) + .header("Content-Type", "application/json") + .json(&mapping) + .send() + .await + .map_err(|e| TestBinaryError::FixtureSetup { + fixture_type: "OpenSearchOps".to_string(), + message: format!("Failed to create index: {e}"), + })?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(TestBinaryError::FixtureSetup { + fixture_type: "OpenSearchOps".to_string(), + message: format!("Failed to create index: status={status}, body={body}"), + }); + } + + info!("Created OpenSearch index: {index_name}"); + Ok(()) + } + } + + fn index_document( + &self, + index_name: &str, + doc_id: &str, + document: &serde_json::Value, + ) -> impl std::future::Future> + Send { + async move { + let url = format!( + "{}/{}/_doc/{}", + self.container().base_url, + index_name, + doc_id + ); + + let response = self + .http_client() + .put(&url) + .header("Content-Type", "application/json") + .json(document) + .send() + .await + .map_err(|e| TestBinaryError::FixtureSetup { + fixture_type: "OpenSearchOps".to_string(), + message: format!("Failed to index document: {e}"), + })?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(TestBinaryError::FixtureSetup { + fixture_type: "OpenSearchOps".to_string(), + message: format!("Failed to index document: status={status}, body={body}"), + }); + } + + Ok(()) + } + } + + fn refresh_index( + &self, + index_name: &str, + ) -> impl std::future::Future> + Send { + async move { + let url = format!("{}/{}/_refresh", self.container().base_url, index_name); + + let response = self.http_client().post(&url).send().await.map_err(|e| { + TestBinaryError::InvalidState { + message: format!("Failed to refresh index: {e}"), + } + })?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(TestBinaryError::InvalidState { + message: format!("Failed to refresh index: status={status}, body={body}"), + }); + } + + info!("Refreshed OpenSearch index: {index_name}"); + Ok(()) + } + } + + #[allow(dead_code)] + fn search_all( + &self, + index_name: &str, + ) -> impl std::future::Future> + Send + { + async move { + let url = format!("{}/{}/_search", self.container().base_url, index_name); + let query = serde_json::json!({ + "query": { "match_all": {} }, + "size": 1000, + "_source": true + }); + + let response = self + .http_client() + .post(&url) + .header("Content-Type", "application/json") + .json(&query) + .send() + .await + .map_err(|e| TestBinaryError::InvalidState { + message: format!("Failed to search index: {e}"), + })?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(TestBinaryError::InvalidState { + message: format!("Failed to search index: status={status}, body={body}"), + }); + } + + let text = response + .text() + .await + .map_err(|e| TestBinaryError::InvalidState { + message: format!("Failed to get response text: {e}"), + })?; + + info!("OpenSearch search response: {text}"); + + serde_json::from_str::(&text).map_err(|e| { + TestBinaryError::InvalidState { + message: format!("Failed to parse search response: {e}, body: {text}"), + } + }) + } + } + + fn count_documents( + &self, + index_name: &str, + ) -> impl std::future::Future> + Send { + async move { + let url = format!("{}/{}/_count", self.container().base_url, index_name); + + let response = self.http_client().get(&url).send().await.map_err(|e| { + TestBinaryError::InvalidState { + message: format!("Failed to count documents: {e}"), + } + })?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(TestBinaryError::InvalidState { + message: format!("Failed to count documents: status={status}, body={body}"), + }); + } + + #[derive(Deserialize)] + struct CountResponse { + count: usize, + } + + let count_response = response.json::().await.map_err(|e| { + TestBinaryError::InvalidState { + message: format!("Failed to parse count response: {e}"), + } + })?; + + Ok(count_response.count) + } + } +} diff --git a/core/integration/tests/connectors/fixtures/opensearch/mod.rs b/core/integration/tests/connectors/fixtures/opensearch/mod.rs new file mode 100644 index 0000000000..963cbfb305 --- /dev/null +++ b/core/integration/tests/connectors/fixtures/opensearch/mod.rs @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +pub mod container; +pub mod source; + +pub use source::{OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture}; diff --git a/core/integration/tests/connectors/fixtures/opensearch/source.rs b/core/integration/tests/connectors/fixtures/opensearch/source.rs new file mode 100644 index 0000000000..3b951a1ec0 --- /dev/null +++ b/core/integration/tests/connectors/fixtures/opensearch/source.rs @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use super::container::{ + DEFAULT_TEST_STREAM, DEFAULT_TEST_TOPIC, ENV_SOURCE_BATCH_SIZE, ENV_SOURCE_INDEX, + ENV_SOURCE_PATH, ENV_SOURCE_POLLING_INTERVAL, ENV_SOURCE_STREAMS_0_SCHEMA, + ENV_SOURCE_STREAMS_0_STREAM, ENV_SOURCE_STREAMS_0_TOPIC, ENV_SOURCE_TIMESTAMP_FIELD, + ENV_SOURCE_URL, OpenSearchContainer, OpenSearchOps, create_http_client, +}; +use async_trait::async_trait; +use iggy_common::IggyTimestamp; +use integration::harness::{TestBinaryError, TestFixture}; +use reqwest_middleware::ClientWithMiddleware as HttpClient; +use std::collections::HashMap; +use uuid::Uuid; + +const TEST_INDEX_PREFIX: &str = "test_documents"; + +/// OpenSearch source fixture for basic document polling. +pub struct OpenSearchSourceFixture { + container: OpenSearchContainer, + http_client: HttpClient, + // Unique per fixture so tests sharing one container never collide on the + // same index. The connector reads from here via ENV_SOURCE_INDEX. + index: String, +} + +impl OpenSearchOps for OpenSearchSourceFixture { + fn container(&self) -> &OpenSearchContainer { + &self.container + } + + fn http_client(&self) -> &HttpClient { + &self.http_client + } +} + +impl OpenSearchSourceFixture { + #[allow(dead_code)] + pub fn index_name(&self) -> &str { + &self.index + } + + pub async fn setup_index(&self) -> Result<(), TestBinaryError> { + self.create_index(&self.index).await + } + + pub async fn insert_document( + &self, + doc_id: i32, + name: &str, + value: i32, + ) -> Result<(), TestBinaryError> { + let timestamp = IggyTimestamp::now().to_rfc3339_string(); + let document = serde_json::json!({ + "id": doc_id, + "name": name, + "value": value, + "timestamp": timestamp + }); + self.index_document(&self.index, &doc_id.to_string(), &document) + .await + } + + pub async fn insert_documents(&self, count: usize) -> Result<(), TestBinaryError> { + for i in 1..=count { + self.insert_document(i as i32, &format!("doc_{i}"), (i * 10) as i32) + .await?; + } + self.refresh_index().await?; + Ok(()) + } + + pub async fn get_document_count(&self) -> Result { + self.count_documents(&self.index).await + } + + pub async fn refresh_index(&self) -> Result<(), TestBinaryError> { + OpenSearchOps::refresh_index(self, &self.index).await + } +} + +#[async_trait] +impl TestFixture for OpenSearchSourceFixture { + async fn setup() -> Result { + let container = OpenSearchContainer::start().await?; + let http_client = create_http_client(); + let index = format!("{TEST_INDEX_PREFIX}_{}", Uuid::new_v4().simple()); + + // Container startup already waits for /_cluster/health to return 200 + // via HttpWaitStrategy, so no additional health check is needed. + Ok(Self { + container, + http_client, + index, + }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + let mut envs = HashMap::new(); + envs.insert(ENV_SOURCE_URL.to_string(), self.container.base_url.clone()); + envs.insert(ENV_SOURCE_INDEX.to_string(), self.index.clone()); + envs.insert(ENV_SOURCE_POLLING_INTERVAL.to_string(), "100ms".to_string()); + envs.insert(ENV_SOURCE_BATCH_SIZE.to_string(), "100".to_string()); + envs.insert( + ENV_SOURCE_TIMESTAMP_FIELD.to_string(), + "timestamp".to_string(), + ); + envs.insert( + ENV_SOURCE_STREAMS_0_STREAM.to_string(), + DEFAULT_TEST_STREAM.to_string(), + ); + envs.insert( + ENV_SOURCE_STREAMS_0_TOPIC.to_string(), + DEFAULT_TEST_TOPIC.to_string(), + ); + envs.insert(ENV_SOURCE_STREAMS_0_SCHEMA.to_string(), "json".to_string()); + envs.insert( + ENV_SOURCE_PATH.to_string(), + "../../target/debug/libiggy_connector_opensearch_source".to_string(), + ); + envs + } +} + +/// OpenSearch source fixture with pre-created index. +pub struct OpenSearchSourcePreCreatedFixture { + inner: OpenSearchSourceFixture, +} + +impl std::ops::Deref for OpenSearchSourcePreCreatedFixture { + type Target = OpenSearchSourceFixture; + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl OpenSearchOps for OpenSearchSourcePreCreatedFixture { + fn container(&self) -> &OpenSearchContainer { + &self.inner.container + } + + fn http_client(&self) -> &HttpClient { + &self.inner.http_client + } +} + +#[async_trait] +impl TestFixture for OpenSearchSourcePreCreatedFixture { + async fn setup() -> Result { + let inner = OpenSearchSourceFixture::setup().await?; + + inner.setup_index().await?; + + Ok(Self { inner }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + self.inner.connectors_runtime_envs() + } +} + +/// OpenSearch source fixture pointing at an index that is never created, +/// for exercising the connector's "missing index" failure path. +pub struct OpenSearchSourceMissingIndexFixture { + inner: OpenSearchSourceFixture, +} + +#[async_trait] +impl TestFixture for OpenSearchSourceMissingIndexFixture { + async fn setup() -> Result { + let inner = OpenSearchSourceFixture::setup().await?; + Ok(Self { inner }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + self.inner.connectors_runtime_envs() + } +} diff --git a/core/integration/tests/connectors/mod.rs b/core/integration/tests/connectors/mod.rs index bb4bcc69f1..d014f7a599 100644 --- a/core/integration/tests/connectors/mod.rs +++ b/core/integration/tests/connectors/mod.rs @@ -27,6 +27,7 @@ mod http_config_provider; mod iceberg; mod influxdb; mod mongodb; +mod opensearch; mod postgres; mod quickwit; mod random; diff --git a/core/integration/tests/connectors/opensearch/mod.rs b/core/integration/tests/connectors/opensearch/mod.rs new file mode 100644 index 0000000000..742b93ee3e --- /dev/null +++ b/core/integration/tests/connectors/opensearch/mod.rs @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +mod opensearch_source; + +const TEST_MESSAGE_COUNT: usize = 3; +const POLL_ATTEMPTS: usize = 100; +const POLL_INTERVAL_MS: u64 = 50; diff --git a/core/integration/tests/connectors/opensearch/opensearch_source.rs b/core/integration/tests/connectors/opensearch/opensearch_source.rs new file mode 100644 index 0000000000..71d6928a11 --- /dev/null +++ b/core/integration/tests/connectors/opensearch/opensearch_source.rs @@ -0,0 +1,367 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use super::{POLL_ATTEMPTS, POLL_INTERVAL_MS, TEST_MESSAGE_COUNT}; +use crate::connectors::fixtures::{ + OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture, +}; +use iggy_common::MessageClient; +use iggy_common::{Consumer, Identifier, PollingStrategy}; +use iggy_connector_sdk::api::{ConnectorStatus, SourceInfoResponse}; +use integration::harness::seeds; +use integration::iggy_harness; +use reqwest::Client; +use std::time::Duration; +use tokio::time::sleep; + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), + seed = seeds::connector_stream +)] +async fn opensearch_source_produces_messages_to_iggy( + harness: &TestHarness, + fixture: OpenSearchSourcePreCreatedFixture, +) { + let client = harness.root_client().await.unwrap(); + + fixture + .insert_documents(TEST_MESSAGE_COUNT) + .await + .expect("Failed to insert documents"); + + let doc_count = fixture + .get_document_count() + .await + .expect("Failed to get document count"); + assert_eq!( + doc_count, TEST_MESSAGE_COUNT, + "Expected {TEST_MESSAGE_COUNT} documents in OpenSearch" + ); + + let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); + let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); + let consumer_id: Identifier = "test_consumer".try_into().unwrap(); + + let mut received: Vec = Vec::new(); + for _ in 0..POLL_ATTEMPTS { + if let Ok(polled) = client + .poll_messages( + &stream_id, + &topic_id, + None, + &Consumer::new(consumer_id.clone()), + &PollingStrategy::next(), + 10, + true, + ) + .await + { + for msg in polled.messages { + if let Ok(json) = serde_json::from_slice(&msg.payload) { + received.push(json); + } + } + if received.len() >= TEST_MESSAGE_COUNT { + break; + } + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + + assert!( + received.len() >= TEST_MESSAGE_COUNT, + "Expected at least {TEST_MESSAGE_COUNT} messages, got {}", + received.len() + ); + + for (i, record) in received.iter().enumerate() { + let expected_id = (i + 1) as i64; + let expected_name = format!("doc_{}", i + 1); + + assert_eq!( + record.get("id").and_then(|v| v.as_i64()), + Some(expected_id), + "ID mismatch at record {i}" + ); + assert_eq!( + record.get("name").and_then(|v| v.as_str()), + Some(expected_name.as_str()), + "Name mismatch at record {i}" + ); + } +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), + seed = seeds::connector_stream +)] +async fn opensearch_source_handles_empty_index( + harness: &TestHarness, + fixture: OpenSearchSourcePreCreatedFixture, +) { + let client = harness.root_client().await.unwrap(); + + let doc_count = fixture + .get_document_count() + .await + .expect("Failed to get document count"); + assert_eq!(doc_count, 0, "Expected empty index"); + + let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); + let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); + let consumer_id: Identifier = "test_consumer".try_into().unwrap(); + + sleep(Duration::from_millis(100)).await; + + let polled = client + .poll_messages( + &stream_id, + &topic_id, + None, + &Consumer::new(consumer_id), + &PollingStrategy::next(), + 10, + false, + ) + .await; + + assert!( + polled.is_ok(), + "Should be able to poll from topic even with empty source" + ); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), + seed = seeds::connector_stream +)] +async fn opensearch_source_produces_bulk_messages( + harness: &TestHarness, + fixture: OpenSearchSourcePreCreatedFixture, +) { + let client = harness.root_client().await.unwrap(); + let bulk_count = 10; + + fixture + .insert_documents(bulk_count) + .await + .expect("Failed to insert documents"); + + let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); + let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); + let consumer_id: Identifier = "test_consumer".try_into().unwrap(); + + let mut received: Vec = Vec::new(); + for _ in 0..POLL_ATTEMPTS { + if let Ok(polled) = client + .poll_messages( + &stream_id, + &topic_id, + None, + &Consumer::new(consumer_id.clone()), + &PollingStrategy::next(), + 100, + true, + ) + .await + { + for msg in polled.messages { + if let Ok(json) = serde_json::from_slice(&msg.payload) { + received.push(json); + } + } + if received.len() >= bulk_count { + break; + } + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + + assert!( + received.len() >= bulk_count, + "Expected at least {bulk_count} messages, got {}", + received.len() + ); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), + seed = seeds::connector_stream +)] +async fn state_persists_across_connector_restart( + harness: &mut TestHarness, + fixture: OpenSearchSourcePreCreatedFixture, +) { + fixture + .insert_documents(TEST_MESSAGE_COUNT) + .await + .expect("Failed to insert first batch"); + + let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); + let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); + let consumer_id: Identifier = "state_test_consumer".try_into().unwrap(); + + let client = harness.root_client().await.unwrap(); + let received_before = { + let mut received: Vec = Vec::new(); + for _ in 0..POLL_ATTEMPTS { + if let Ok(polled) = client + .poll_messages( + &stream_id, + &topic_id, + None, + &Consumer::new(consumer_id.clone()), + &PollingStrategy::next(), + 10, + true, + ) + .await + { + for msg in polled.messages { + if let Ok(json) = serde_json::from_slice(&msg.payload) { + received.push(json); + } + } + if received.len() >= TEST_MESSAGE_COUNT { + break; + } + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + received + }; + assert_eq!(received_before.len(), TEST_MESSAGE_COUNT); + + harness + .server_mut() + .stop_dependents() + .expect("Failed to stop connectors"); + + let second_batch_start_id = (TEST_MESSAGE_COUNT + 1) as i32; + for i in 0..TEST_MESSAGE_COUNT { + fixture + .insert_document( + second_batch_start_id + i as i32, + &format!("doc_batch2_{i}"), + (TEST_MESSAGE_COUNT + i) as i32 * 10, + ) + .await + .expect("Failed to insert document"); + } + fixture + .refresh_index() + .await + .expect("Failed to refresh index"); + + harness + .server_mut() + .start_dependents() + .await + .expect("Failed to restart connectors"); + sleep(Duration::from_millis(100)).await; + + let mut received_after: Vec = Vec::new(); + for _ in 0..POLL_ATTEMPTS { + if let Ok(polled) = client + .poll_messages( + &stream_id, + &topic_id, + None, + &Consumer::new(consumer_id.clone()), + &PollingStrategy::next(), + 10, + true, + ) + .await + { + for msg in polled.messages { + if let Ok(json) = serde_json::from_slice(&msg.payload) { + received_after.push(json); + } + } + if received_after.len() >= TEST_MESSAGE_COUNT { + break; + } + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + + assert_eq!(received_after.len(), TEST_MESSAGE_COUNT); + + for record in &received_after { + let id = record.get("id").and_then(|v| v.as_i64()).unwrap_or(0); + assert!( + id > TEST_MESSAGE_COUNT as i64, + "After restart, got ID {id} from first batch" + ); + } +} + +async fn fetch_sources(http_client: &Client, api_address: &str) -> Vec { + let response = http_client + .get(format!("{api_address}/sources")) + .send() + .await + .expect("Failed to query /sources"); + assert_eq!(response.status(), 200); + response.json().await.expect("Failed to parse sources") +} + +/// Negative path: the configured index does not exist, so `open()` must +/// fail with a `Storage` error and the runtime reports the source as +/// `ConnectorStatus::Error` without aborting. +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), + seed = seeds::connector_stream +)] +async fn opensearch_source_with_missing_index_reports_error( + harness: &TestHarness, + _fixture: OpenSearchSourceMissingIndexFixture, +) { + let api_address = harness + .connectors_runtime() + .expect("connector runtime should be available") + .http_url(); + let http_client = Client::new(); + + let mut sources = fetch_sources(&http_client, &api_address).await; + for _ in 0..POLL_ATTEMPTS { + if sources + .iter() + .any(|source| source.status == ConnectorStatus::Error) + { + break; + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + sources = fetch_sources(&http_client, &api_address).await; + } + + assert_eq!(sources.len(), 1, "Expected a single configured source"); + let source = &sources[0]; + assert_eq!(source.status, ConnectorStatus::Error); + let last_error = source + .last_error + .as_ref() + .expect("Source with missing index should expose a last_error"); + assert!( + last_error.message.contains("does not exist"), + "last_error should mention the missing index, got: {}", + last_error.message + ); +} diff --git a/core/integration/tests/connectors/opensearch/source.toml b/core/integration/tests/connectors/opensearch/source.toml new file mode 100644 index 0000000000..f0baa94e43 --- /dev/null +++ b/core/integration/tests/connectors/opensearch/source.toml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[connectors] +config_type = "local" +config_dir = "../connectors/sources/opensearch_source" From b4a182ee00cb0dbd2af6ae5131eca09afbfb8409 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Tue, 16 Jun 2026 12:51:20 -0400 Subject: [PATCH 04/19] Refactor OpenSearch source state, search and docs , unit test cases and code cleanup Refactors the OpenSearch source connector: replaces legacy state handling with a clearer restore_state flow that rejects corrupt runtime state, introduces validation for open() config, and centralises file-backed state storage creation (create_state_storage). Implements search_after cursoring with combined (timestamp_field, _id) sort, robust document timestamp parsing (RFC3339 or epoch), defaults for polling interval and batch size, and optional verbose logging. Cleans up and simplifies state structs (removes scroll/offset fields), improves error messages formatting, and uses parse_duration for interval parsing. Also updates docs and metadata: rewrites the connector README to focus on usage and state semantics, adds a dependencies.md listing runtime deps, updates Cargo.toml to remove simd-json and adjust ignored list, and adds a docker-compose.yml for integration tests. Tests and fixtures were updated to reflect the new cursor/state behaviour and naming conventions. --- Cargo.lock | 1 - .../sources/opensearch_source/Cargo.toml | 7 +- .../sources/opensearch_source/README.md | 215 ++------- .../sources/opensearch_source/dependencies.md | 46 ++ .../sources/opensearch_source/src/lib.rs | 450 ++++++++++-------- .../opensearch_source/src/state_manager.rs | 270 ++--------- .../connectors/fixtures/opensearch/source.rs | 5 +- .../connectors/opensearch/docker-compose.yml | 44 ++ .../opensearch/opensearch_source.rs | 48 +- 9 files changed, 475 insertions(+), 611 deletions(-) create mode 100644 core/connectors/sources/opensearch_source/dependencies.md create mode 100644 core/integration/tests/connectors/opensearch/docker-compose.yml diff --git a/Cargo.lock b/Cargo.lock index f3fec351ed..b452668d51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6947,7 +6947,6 @@ dependencies = [ "secrecy", "serde", "serde_json", - "simd-json", "tokio", "tracing", ] diff --git a/core/connectors/sources/opensearch_source/Cargo.toml b/core/connectors/sources/opensearch_source/Cargo.toml index 5c91507112..0dfb5f5290 100644 --- a/core/connectors/sources/opensearch_source/Cargo.toml +++ b/core/connectors/sources/opensearch_source/Cargo.toml @@ -29,8 +29,12 @@ repository = "https://github.com/apache/iggy" readme = "../../README.md" publish = false +# dashmap and once_cell are not imported directly in this crate's source, but +# the source_connector! macro (in iggy_connector_sdk::source) expands bare +# `use dashmap::DashMap` and `use once_cell::sync::Lazy` into this crate's +# namespace, so they must be listed here. [package.metadata.cargo-machete] -ignored = ["dashmap", "once_cell", "futures", "simd-json"] +ignored = ["dashmap", "once_cell"] [lib] crate-type = ["cdylib", "lib"] @@ -47,6 +51,5 @@ rmp-serde = { workspace = true } secrecy = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -simd-json = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } diff --git a/core/connectors/sources/opensearch_source/README.md b/core/connectors/sources/opensearch_source/README.md index 466b20f7b4..f2c19a9f6f 100644 --- a/core/connectors/sources/opensearch_source/README.md +++ b/core/connectors/sources/opensearch_source/README.md @@ -1,22 +1,19 @@ -# OpenSearch Source Connector with State Management +# OpenSearch Source Connector -This OpenSearch source connector provides comprehensive state management capabilities to track processing progress and enable fault-tolerant data ingestion. +Polls documents from an OpenSearch index and publishes them to Iggy streams as JSON +messages. Incremental progress is tracked with OpenSearch `search_after` pagination +on `(timestamp_field, _id)`. ## Features -- **Incremental Data Processing**: Track last processed timestamp to avoid reprocessing data -- **Cursor-based Pagination**: Support for document ID-based cursors -- **Scroll-based Pagination**: Support for OpenSearch scroll API -- **Error Tracking**: Monitor error counts and last error messages -- **Processing Statistics**: Track performance metrics and processing times -- **Persistent State Storage**: Multiple storage backends (file, OpenSearch, Redis) -- **Auto-save**: Configurable automatic state persistence -- **State Recovery**: Resume processing from last known position after restart +- Incremental polling via `search_after` on a configured timestamp field plus `_id` +- Optional custom OpenSearch query (`match_all` by default) +- Basic authentication (username + password) +- Runtime state persistence via the connectors runtime (`ConnectorState` / MessagePack) +- Optional supplementary file-backed state when `plugin_config.state.enabled = true` ## Configuration -### Basic Configuration - ```toml type = "source" key = "opensearch" @@ -24,6 +21,7 @@ enabled = true version = 0 name = "OpenSearch source" path = "target/release/libiggy_connector_opensearch_source" +plugin_config_format = "json" [[streams]] stream = "opensearch_stream" @@ -38,174 +36,65 @@ index = "logs-*" polling_interval = "30s" batch_size = 100 timestamp_field = "@timestamp" -query = { - "match_all": {} -} +query = { "match_all": {} } ``` -### State Management Configuration - -```toml -[plugin_config] -# ... basic config ... -state = { - enabled = true - storage_type = "file" # "file", "opensearch", "redis" - storage_config = { - base_path = "./connector_states" # for file storage - # index = "connector_states" # for opensearch storage - # url = "redis://localhost:6379" # for redis storage - } - state_id = "opensearch_logs_connector" - auto_save_interval = "5m" - tracked_fields = [ - "last_poll_timestamp", - "last_document_id", - "total_documents_fetched" - ] -} -``` - -## State Information - -The connector tracks the following state information: +### Required fields -### Processing State +| Field | Description | +| --- | --- | +| `url` | OpenSearch HTTP endpoint | +| `index` | Index name or pattern | +| `timestamp_field` | Document field used for sort order and incremental cursors (required) | -- `last_poll_timestamp`: Last successful poll timestamp -- `total_documents_fetched`: Total number of documents processed -- `poll_count`: Number of polling cycles executed -- `last_document_id`: Last processed document ID (for cursor pagination) -- `last_scroll_id`: Last scroll ID (for scroll pagination) -- `last_offset`: Last processed offset +### Optional fields -### Error Tracking +| Field | Default | Description | +| --- | --- | --- | +| `polling_interval` | `10s` | Delay before each poll (humantime format) | +| `batch_size` | `100` | Maximum documents per search request (minimum `1`) | +| `query` | `match_all` | OpenSearch query DSL object | +| `username` / `password` | none | HTTP basic authentication | +| `verbose_logging` | `false` | Emit per-poll batch counts at `info!` instead of `debug!` | -- `error_count`: Total number of errors encountered -- `last_error`: Last error message +### File-backed state (optional) -### Performance Statistics - -- `total_bytes_processed`: Total bytes processed -- `avg_batch_processing_time_ms`: Average processing time per batch -- `last_successful_poll`: Timestamp of last successful poll -- `empty_polls_count`: Number of polls that returned no documents -- `successful_polls_count`: Number of successful polls - -## Storage Backends - -### File Storage (Default) +Runtime state is always returned from `poll()` and persisted by the connectors +runtime. To additionally mirror state to JSON files on disk: ```toml -state = { - enabled = true - storage_type = "file" - storage_config = { - base_path = "./connector_states" - } -} +[plugin_config.state] +enabled = true +storage_type = "file" +storage_config = { base_path = "./connector_states" } +state_id = "opensearch_logs_connector" ``` -### OpenSearch Storage +Only `storage_type = "file"` is implemented. State is saved on `close()` when +`state.enabled = true`. -```toml -state = { - enabled = true - storage_type = "opensearch" - storage_config = { - index = "connector_states" - url = "http://localhost:9200" - } -} -``` +## State fields -### Redis Storage +The connector tracks: -```toml -state = { - enabled = true - storage_type = "redis" - storage_config = { - url = "redis://localhost:6379" - key_prefix = "connector_states:" - } -} -``` +- `search_after`: OpenSearch sort tuple from the last document in the previous batch +- `last_poll_timestamp`: timestamp of the last processed document +- `last_document_id`: `_id` of the last processed document +- `total_documents_fetched`, `poll_count`, error counters, and processing statistics -## State File Format - -State files are stored as JSON with the following structure: - -```json -{ - "id": "opensearch_logs_connector", - "last_updated": "2024-01-15T10:30:00Z", - "version": 1, - "data": { - "last_poll_timestamp": "2024-01-15T10:30:00Z", - "total_documents_fetched": 15000, - "poll_count": 150, - "last_document_id": "doc_12345", - "last_scroll_id": "scroll_abc123", - "last_offset": 15000, - "error_count": 2, - "last_error": "Connection timeout", - "processing_stats": { - "total_bytes_processed": 1048576, - "avg_batch_processing_time_ms": 125.5, - "last_successful_poll": "2024-01-15T10:30:00Z", - "empty_polls_count": 5, - "successful_polls_count": 145 - } - }, - "metadata": { - "connector_type": "opensearch_source", - "connector_id": 1, - "index": "logs-*", - "url": "http://localhost:9200" - } -} -``` +Corrupt runtime state causes `open()` to fail with `InitError` rather than silently +resetting the cursor. -## Best Practices +## Timestamp formats -1. **State ID Uniqueness**: Use unique state IDs for different connector instances -2. **Auto-save Interval**: Set appropriate auto-save intervals based on your data volume -3. **Storage Location**: Use persistent storage locations for production deployments -4. **State Cleanup**: Regularly clean up old state files to prevent disk space issues -5. **Error Handling**: Monitor error counts and implement appropriate alerting -6. **Backup**: Regularly backup state files for disaster recovery +The configured `timestamp_field` may be an RFC 3339 string or epoch milliseconds / +seconds in the document `_source`. ## Troubleshooting -### Common Issues - -1. **State Not Loading**: Check file permissions and storage path -2. **State Corruption**: Delete corrupted state files to start fresh -3. **Performance Issues**: Adjust auto-save interval and batch sizes -4. **Storage Full**: Implement state cleanup policies - -### Monitoring - -Monitor the following metrics: - -- State save/load success rates -- Processing statistics -- Error counts and types -- Storage usage for state files - -## Migration - -To migrate from a connector without state management: - -1. Add state configuration to your connector config -2. Set `enabled = true` in state config -3. Restart the connector -4. The connector will start tracking state from the next poll cycle - -To migrate between storage backends: - -1. Export state from current storage -2. Update storage configuration -3. Import state to new storage -4. Restart connector +| Symptom | Check | +| --- | --- | +| `open()` fails with missing index | Index name, URL, and credentials | +| `open()` fails with `state restore failed` | Delete or repair the connector runtime state file | +| No new documents after restart | `timestamp_field` mapping must match indexed documents | +| Duplicate messages | Lower `batch_size` only after confirming sort stability on `(timestamp_field, _id)` | diff --git a/core/connectors/sources/opensearch_source/dependencies.md b/core/connectors/sources/opensearch_source/dependencies.md new file mode 100644 index 0000000000..fc61ce8dd2 --- /dev/null +++ b/core/connectors/sources/opensearch_source/dependencies.md @@ -0,0 +1,46 @@ + + +# OpenSearch Source Connector — Direct Runtime Dependencies + +This file lists every direct (non-dev) dependency declared in +`core/connectors/sources/opensearch_source/Cargo.toml`, together with +its workspace-pinned version, license, and the specific role it plays +in this connector. Transitive dependencies are not listed here; refer +to `cargo tree -p iggy_connector_opensearch_source` for the full graph. + +--- + +## Runtime dependencies + +| Crate | Version (workspace) | License | Role in this connector | +| --- | --- | --- | --- | +| `async-trait` | `^0.1.89` | MIT / Apache-2.0 | Proc-macro that enables `async fn` in trait definitions; required by the `Source` trait impl in `lib.rs`. | +| `dashmap` | `^6.1.0` | MIT | Concurrent hash map; injected into this crate's namespace by the `source_connector!` macro expansion in the SDK. Not used directly in source files. | +| `humantime` | `^2.3.0` | MIT / Apache-2.0 | Parses human-readable duration strings in optional file-state `auto_save_interval` config (reserved for future use). | +| `iggy_common` | workspace | Apache-2.0 | Shared Iggy types: `DateTime`, `Utc`, and `serde_secret` for optional basic-auth password serialisation. | +| `iggy_connector_sdk` | workspace | Apache-2.0 | Core connector abstractions: `Source` trait, `ProducedMessage`, `ProducedMessages`, `ConnectorState`, `Schema`, `Error`, `parse_duration`, and the `source_connector!` registration macro. | +| `once_cell` | `^1.21.4` | MIT / Apache-2.0 | `Lazy` global; injected by the `source_connector!` macro expansion in the SDK. Not used directly in source files. | +| `opensearch` | `2.4.0` | Apache-2.0 | Official OpenSearch Rust client for index existence checks and `search` requests with `search_after` pagination. | +| `rmp-serde` | workspace | MIT / Apache-2.0 | Serialises connector runtime state to MessagePack for the connectors runtime state file. | +| `secrecy` | `^0.10` | MIT / Apache-2.0 | `SecretString` wrapper that prevents accidental logging of passwords in config structs. | +| `serde` | workspace | MIT / Apache-2.0 | Derive macros for config and persisted-state de/serialisation. | +| `serde_json` | workspace | MIT / Apache-2.0 | Builds OpenSearch query bodies and serialises document payloads into produced messages. | +| `tokio` | workspace | MIT | Async runtime; `tokio::sync::Mutex` for connector state and `tokio::time::sleep` for poll-interval delays. | +| `tracing` | workspace | MIT | Structured logging macros (`info!`, `warn!`, `error!`) used throughout the poll loop and state paths. | diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs index 498d0bf386..eea90ce874 100644 --- a/core/connectors/sources/opensearch_source/src/lib.rs +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -18,6 +18,7 @@ */ use async_trait::async_trait; use iggy_common::{DateTime, Utc}; +use iggy_connector_sdk::retry::parse_duration; use iggy_connector_sdk::{ ConnectorState, Error, ProducedMessage, ProducedMessages, Schema, Source, source_connector, }; @@ -29,65 +30,49 @@ use opensearch::{ use secrecy::{ExposeSecret, SecretString}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; -use std::str::FromStr; -use std::sync::Arc; use std::time::Duration; use tokio::{sync::Mutex, time::sleep}; -use tracing::{info, warn}; +use tracing::{debug, error, info, warn}; mod state_manager; -use crate::state_manager::{FileStateStorage, SourceState, StateStorage}; -pub use state_manager::{StateInfo, StateManager, StateStats}; +use crate::state_manager::{SourceState, create_state_storage}; source_connector!(OpenSearchSource); const CONNECTOR_NAME: &str = "OpenSearch source"; +const DEFAULT_POLLING_INTERVAL: &str = "10s"; +const DEFAULT_BATCH_SIZE: usize = 100; #[derive(Debug, Clone, Serialize, Deserialize)] struct State { last_poll_timestamp: Option>, total_documents_fetched: usize, poll_count: usize, - /// Last document ID processed (for cursor-based pagination) last_document_id: Option, - /// Last scroll ID (for scroll-based pagination) - last_scroll_id: Option, - /// Last processed offset - last_offset: Option, - /// Error count and last error + /// OpenSearch `search_after` tuple from the last hit in the previous batch. + search_after: Option>, error_count: usize, last_error: Option, - /// Processing statistics processing_stats: ProcessingStats, } #[derive(Debug, Clone, Serialize, Deserialize)] struct ProcessingStats { - /// Total bytes processed total_bytes_processed: u64, - /// Average processing time per batch avg_batch_processing_time_ms: f64, - /// Last successful processing timestamp last_successful_poll: Option>, - /// Number of empty polls empty_polls_count: usize, - /// Number of successful polls successful_polls_count: usize, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StateConfig { - /// Enable state persistence + #[serde(default)] pub enabled: bool, - /// State storage type: "file", "opensearch", "redis", etc. pub storage_type: Option, - /// State storage configuration (depends on storage_type) pub storage_config: Option, - /// State ID for this connector instance pub state_id: Option, - /// Auto-save state interval (e.g., "30s", "5m") pub auto_save_interval: Option, - /// Fields to track in state (e.g., ["last_timestamp", "last_document_id"]) pub tracked_fields: Option>, } @@ -102,7 +87,7 @@ pub struct OpenSearchSourceConfig { pub polling_interval: Option, pub batch_size: Option, pub timestamp_field: Option, - pub scroll_timeout: Option, + pub verbose_logging: Option, pub state: Option, } @@ -112,51 +97,54 @@ pub struct OpenSearchSource { config: OpenSearchSourceConfig, client: Option, polling_interval: Duration, + search_query: Value, + verbose: bool, state: Mutex, + /// `Some(cause)` when runtime state restore was rejected; `None` means restore succeeded. + state_restore_error: Option, +} + +impl Default for State { + fn default() -> Self { + Self { + last_poll_timestamp: None, + total_documents_fetched: 0, + poll_count: 0, + last_document_id: None, + search_after: None, + error_count: 0, + last_error: None, + processing_stats: ProcessingStats { + total_bytes_processed: 0, + avg_batch_processing_time_ms: 0.0, + last_successful_poll: None, + empty_polls_count: 0, + successful_polls_count: 0, + }, + } + } } impl OpenSearchSource { pub fn new(id: u32, config: OpenSearchSourceConfig, state: Option) -> Self { - let polling_interval = config - .polling_interval - .as_deref() - .unwrap_or("10s") - .parse::() - .unwrap_or_else(|_| humantime::Duration::from_str("10s").unwrap()) - .into(); - - let restored_state = state - .and_then(|s| s.deserialize::(CONNECTOR_NAME, id)) - .inspect(|s| { - info!( - "Restored state for {CONNECTOR_NAME} connector with ID: {id}. \ - Documents fetched: {}, poll count: {}", - s.total_documents_fetched, s.poll_count - ); - }); + let polling_interval = + parse_duration(config.polling_interval.as_deref(), DEFAULT_POLLING_INTERVAL); + let search_query = config + .query + .clone() + .unwrap_or_else(|| json!({ "match_all": {} })); + let verbose = config.verbose_logging.unwrap_or(false); + let (restored_state, state_restore_error) = restore_state(id, state); OpenSearchSource { id, config, client: None, polling_interval, - state: Mutex::new(restored_state.unwrap_or(State { - last_poll_timestamp: None, - total_documents_fetched: 0, - poll_count: 0, - last_document_id: None, - last_scroll_id: None, - last_offset: None, - error_count: 0, - last_error: None, - processing_stats: ProcessingStats { - total_bytes_processed: 0, - avg_batch_processing_time_ms: 0.0, - last_successful_poll: None, - empty_polls_count: 0, - successful_polls_count: 0, - }, - })), + search_query, + verbose, + state: Mutex::new(restored_state), + state_restore_error, } } @@ -164,40 +152,17 @@ impl OpenSearchSource { ConnectorState::serialize(state, CONNECTOR_NAME, self.id) } - /// Create state storage based on configuration - fn create_state_storage(&self) -> Option> { - let state_config = self.config.state.as_ref()?; - if !state_config.enabled { - return None; - } - - match state_config.storage_type.as_deref() { - Some("file") | None => { - let base_path = state_config - .storage_config - .as_ref() - .and_then(|c| c.get("base_path")) - .and_then(|p| p.as_str()) - .unwrap_or("./connector_states"); + fn batch_size(&self) -> usize { + self.config.batch_size.unwrap_or(DEFAULT_BATCH_SIZE) + } - Some(Arc::new(FileStateStorage::new(base_path))) - } - Some("opensearch") => { - // TODO: Implement OpenSearch-based state storage - warn!("OpenSearch state storage not yet implemented, falling back to file storage"); - Some(Arc::new(FileStateStorage::new("./connector_states"))) - } - Some(storage_type) => { - warn!( - "Unknown state storage type: {}, falling back to file storage", - storage_type - ); - Some(Arc::new(FileStateStorage::new("./connector_states"))) - } - } + fn timestamp_field(&self) -> &str { + self.config + .timestamp_field + .as_deref() + .expect("timestamp_field validated at open()") } - /// Get state ID for this connector fn get_state_id(&self) -> String { self.config .state @@ -206,7 +171,6 @@ impl OpenSearchSource { .unwrap_or_else(|| format!("opensearch_source_{}", self.id)) } - /// Convert internal state to SourceState async fn internal_state_to_source_state(&self) -> Result { let state = self.state.lock().await; @@ -215,8 +179,7 @@ impl OpenSearchSource { "total_documents_fetched": state.total_documents_fetched, "poll_count": state.poll_count, "last_document_id": state.last_document_id, - "last_scroll_id": state.last_scroll_id, - "last_offset": state.last_offset, + "search_after": state.search_after, "error_count": state.error_count, "last_error": state.last_error, "processing_stats": state.processing_stats, @@ -236,7 +199,6 @@ impl OpenSearchSource { }) } - /// Convert SourceState to internal state async fn source_state_to_internal_state( &mut self, source_state: SourceState, @@ -264,15 +226,13 @@ impl OpenSearchSource { } if let Some(doc_id) = data.get("last_document_id") { - state.last_document_id = doc_id.as_str().map(|s| s.to_string()); - } - - if let Some(scroll_id) = data.get("last_scroll_id") { - state.last_scroll_id = scroll_id.as_str().map(|s| s.to_string()); + state.last_document_id = doc_id.as_str().map(str::to_owned); } - if let Some(offset) = data.get("last_offset") { - state.last_offset = offset.as_u64(); + if let Some(search_after) = data.get("search_after") + && let Ok(cursor) = serde_json::from_value(search_after.clone()) + { + state.search_after = cursor; } if let Some(error_count) = data.get("error_count") @@ -282,7 +242,7 @@ impl OpenSearchSource { } if let Some(last_error) = data.get("last_error") { - state.last_error = last_error.as_str().map(|s| s.to_string()); + state.last_error = last_error.as_str().map(str::to_owned); } if let Some(stats) = data.get("processing_stats") @@ -310,134 +270,187 @@ impl OpenSearchSource { let transport = transport_builder .build() - .map_err(|e| Error::Storage(format!("Failed to build transport: {}", e)))?; + .map_err(|e| Error::Storage(format!("Failed to build transport: {e}")))?; Ok(OpenSearch::new(transport)) } async fn search_documents(&self, client: &OpenSearch) -> Result, Error> { let state = self.state.lock().await; - let batch_size = self.config.batch_size.unwrap_or(100); - - // Build query based on timestamp field if configured - let mut query = self.config.query.clone().unwrap_or_else(|| { - json!({ - "match_all": {} - }) - }); - - // Add timestamp filter for incremental polling - if let Some(timestamp_field) = &self.config.timestamp_field - && let Some(last_timestamp) = state.last_poll_timestamp - { - query = json!({ - "bool": { - "must": [ - query, - { - "range": { - timestamp_field: { - "gt": last_timestamp.to_rfc3339() - } - } - } - ] - } - }); - } + let batch_size = self.batch_size(); + let timestamp_field = self.timestamp_field(); + let search_after = state.search_after.clone(); + drop(state); - let search_body = json!({ - "query": query, + let mut search_body = json!({ + "query": self.search_query, "size": batch_size, "sort": [ - { - self.config.timestamp_field.as_deref().unwrap_or("@timestamp"): { - "order": "asc" - } - } + { timestamp_field: { "order": "asc" } }, + { "_id": { "order": "asc" } } ] }); - drop(state); + if let Some(cursor) = search_after { + search_body["search_after"] = json!(cursor); + } let response = client .search(SearchParts::Index(&[&self.config.index])) .body(search_body) .send() .await - .map_err(|e| Error::Storage(format!("Failed to execute search: {}", e)))?; + .map_err(|e| Error::Storage(format!("Failed to execute search: {e}")))?; if !response.status_code().is_success() { let error_text = response .text() .await - .unwrap_or_else(|_| "Unknown error".to_string()); + .map_err(|e| Error::Storage(format!("Failed to read search error body: {e}")))?; return Err(Error::Storage(format!( - "Search request failed: {}", - error_text + "Search request failed: {error_text}" ))); } let response_body: Value = response .json() .await - .map_err(|e| Error::Storage(format!("Failed to parse search response: {}", e)))?; - - let mut messages = Vec::new(); - let mut latest_timestamp = None; + .map_err(|e| Error::Storage(format!("Failed to parse search response: {e}")))?; - if let Some(hits) = response_body + let hits = response_body .get("hits") .and_then(|h| h.get("hits")) .and_then(|h| h.as_array()) - { - for hit in hits { - if let Some(source) = hit.get("_source") { - // Extract timestamp for incremental polling - if let Some(timestamp_field) = &self.config.timestamp_field - && let Some(timestamp_str) = - source.get(timestamp_field).and_then(|v| v.as_str()) - && let Ok(timestamp) = DateTime::parse_from_rfc3339(timestamp_str) - { - let timestamp_utc = timestamp.with_timezone(&Utc); - if latest_timestamp.is_none() || timestamp_utc > latest_timestamp.unwrap() { - latest_timestamp = Some(timestamp_utc); - } - } - - // Create message from document - let payload = serde_json::to_vec(source).map_err(|e| { - Error::Serialization(format!("Failed to serialize document: {}", e)) - })?; - - let message = ProducedMessage { - id: None, - headers: None, - checksum: None, - timestamp: None, - origin_timestamp: None, - payload, - }; - messages.push(message); - } + .cloned() + .unwrap_or_default(); + + let mut messages = Vec::with_capacity(hits.len()); + let mut batch_bytes = 0u64; + let mut last_search_after = None; + let mut last_document_id = None; + let mut last_poll_timestamp = None; + + for hit in &hits { + if let Some(sort) = hit.get("sort").and_then(|s| s.as_array()) { + last_search_after = Some(sort.clone()); + } + + if let Some(document_id) = hit.get("_id").and_then(|v| v.as_str()) { + last_document_id = Some(document_id.to_string()); } + + let Some(source) = hit.get("_source") else { + continue; + }; + + if let Some(timestamp_value) = source.get(timestamp_field) + && let Some(timestamp_utc) = parse_document_timestamp(timestamp_value) + { + last_poll_timestamp = Some(timestamp_utc); + } + + let payload = serde_json::to_vec(source) + .map_err(|e| Error::Serialization(format!("Failed to serialize document: {e}")))?; + batch_bytes += payload.len() as u64; + + messages.push(ProducedMessage { + id: None, + headers: None, + checksum: None, + timestamp: None, + origin_timestamp: None, + payload, + }); } - // Update state let mut state = self.state.lock().await; state.total_documents_fetched += messages.len(); state.poll_count += 1; - if let Some(timestamp) = latest_timestamp { + state.search_after = last_search_after; + state.last_document_id = last_document_id; + if let Some(timestamp) = last_poll_timestamp { state.last_poll_timestamp = Some(timestamp); } + state.processing_stats.total_bytes_processed += batch_bytes; Ok(messages) } } +fn restore_state(id: u32, state: Option) -> (State, Option) { + let Some(connector_state) = state else { + return (State::default(), None); + }; + + let bytes = connector_state.0; + match ConnectorState(bytes.clone()).deserialize::(CONNECTOR_NAME, id) { + Some(restored) => { + info!( + "Restored state for {CONNECTOR_NAME} connector with ID: {id}. \ + Documents fetched: {}, poll count: {}", + restored.total_documents_fetched, restored.poll_count + ); + (restored, None) + } + None => { + let cause = "persisted state exists but could not be deserialized. \ + Refusing to start to prevent silent cursor reset." + .to_string(); + error!("{CONNECTOR_NAME} ID {id}: {cause}"); + (State::default(), Some(cause)) + } + } +} + +fn validate_open_config(config: &OpenSearchSourceConfig) -> Result<(), Error> { + if config.timestamp_field.as_deref().is_none_or(str::is_empty) { + return Err(Error::InvalidConfigValue( + "timestamp_field is required for incremental OpenSearch polling".to_string(), + )); + } + + if matches!(config.batch_size, Some(0)) { + return Err(Error::InvalidConfigValue( + "batch_size must be at least 1".to_string(), + )); + } + + if let Some(state) = &config.state + && state.enabled + { + create_state_storage(state)?; + } + + Ok(()) +} + +fn parse_document_timestamp(value: &Value) -> Option> { + match value { + Value::String(text) => DateTime::parse_from_rfc3339(text) + .ok() + .map(|timestamp| timestamp.with_timezone(&Utc)), + Value::Number(number) => { + let raw = number.as_i64()?; + let millis = if raw.abs() > 1_000_000_000_000 { + raw + } else { + raw.saturating_mul(1_000) + }; + DateTime::from_timestamp_millis(millis).map(|timestamp| timestamp.with_timezone(&Utc)) + } + _ => None, + } +} + #[async_trait] impl Source for OpenSearchSource { async fn open(&mut self) -> Result<(), Error> { + if let Some(ref cause) = self.state_restore_error { + return Err(Error::InitError(format!("state restore failed: {cause}"))); + } + + validate_open_config(&self.config)?; + info!( "Opening OpenSearch source connector with ID: {} for URL: {}, index: {}", self.id, self.config.url, self.config.index @@ -445,7 +458,6 @@ impl Source for OpenSearchSource { let client = self.create_client().await?; - // Test connection by checking if index exists let response = client .indices() .exists(opensearch::indices::IndicesExistsParts::Index(&[&self @@ -453,7 +465,7 @@ impl Source for OpenSearchSource { .index])) .send() .await - .map_err(|e| Error::Storage(format!("Failed to check index existence: {}", e)))?; + .map_err(|e| Error::Storage(format!("Failed to check index existence: {e}")))?; if !response.status_code().is_success() { return Err(Error::Storage(format!( @@ -464,7 +476,6 @@ impl Source for OpenSearchSource { self.client = Some(client); - // Load state if state management is enabled if self .config .state @@ -498,7 +509,6 @@ impl Source for OpenSearchSource { let messages = match self.search_documents(client).await { Ok(msgs) => { - // Update success statistics let mut state = self.state.lock().await; state.processing_stats.successful_polls_count += 1; state.processing_stats.last_successful_poll = Some(Utc::now()); @@ -516,11 +526,27 @@ impl Source for OpenSearchSource { state.processing_stats.empty_polls_count += 1; } + let produced_count = msgs.len(); + let total_documents_fetched = state.total_documents_fetched; drop(state); + + if self.verbose { + info!( + "OpenSearch source connector ID: {} produced {produced_count} messages. \ + Total fetched: {total_documents_fetched}", + self.id + ); + } else { + debug!( + "OpenSearch source connector ID: {} produced {produced_count} messages. \ + Total fetched: {total_documents_fetched}", + self.id + ); + } + msgs } Err(e) => { - // Update error statistics let mut state = self.state.lock().await; state.error_count += 1; state.last_error = Some(e.to_string()); @@ -548,7 +574,6 @@ impl Source for OpenSearchSource { ); drop(state); - // Save final state if state management is enabled if self .config .state @@ -586,7 +611,7 @@ mod tests { polling_interval: Some("100ms".to_string()), batch_size: Some(10), timestamp_field: Some("timestamp".to_string()), - scroll_timeout: None, + verbose_logging: None, state: None, } } @@ -597,8 +622,7 @@ mod tests { total_documents_fetched: 500, poll_count: 5, last_document_id: Some("doc_42".to_string()), - last_scroll_id: None, - last_offset: Some(500), + search_after: Some(vec![json!("2024-01-01T00:00:00Z"), json!("doc_42")]), error_count: 1, last_error: Some("connection reset".to_string()), processing_stats: ProcessingStats { @@ -625,6 +649,7 @@ mod tests { assert_eq!(restored.total_documents_fetched, 500); assert_eq!(restored.poll_count, 5); assert_eq!(restored.last_document_id, Some("doc_42".to_string())); + assert!(source.state_restore_error.is_none()); }); } @@ -638,14 +663,16 @@ mod tests { assert_eq!(state.total_documents_fetched, 0); assert_eq!(state.poll_count, 0); assert_eq!(state.last_document_id, None); + assert!(source.state_restore_error.is_none()); }); } #[test] - fn given_invalid_state_should_start_fresh() { + fn given_invalid_state_should_set_state_restore_error() { let invalid_state = ConnectorState(b"not valid msgpack".to_vec()); let source = OpenSearchSource::new(1, test_config(), Some(invalid_state)); + assert!(source.state_restore_error.is_some()); let runtime = tokio::runtime::Runtime::new().unwrap(); runtime.block_on(async { let state = source.state.lock().await; @@ -655,7 +682,47 @@ mod tests { } #[test] - fn state_should_be_serializable_and_deserializable() { + fn given_invalid_state_when_open_should_fail() { + let invalid_state = ConnectorState(b"not valid msgpack".to_vec()); + let mut source = OpenSearchSource::new(1, test_config(), Some(invalid_state)); + let runtime = tokio::runtime::Runtime::new().unwrap(); + let result = runtime.block_on(source.open()); + assert!( + matches!(result, Err(Error::InitError(_))), + "open() must fail with InitError on restore failure" + ); + } + + #[test] + fn given_missing_timestamp_field_when_validate_should_fail() { + let mut config = test_config(); + config.timestamp_field = None; + let error = validate_open_config(&config).expect_err("missing timestamp_field"); + assert!(matches!(error, Error::InvalidConfigValue(_))); + } + + #[test] + fn given_zero_batch_size_when_validate_should_fail() { + let mut config = test_config(); + config.batch_size = Some(0); + let error = validate_open_config(&config).expect_err("zero batch_size"); + assert!(matches!(error, Error::InvalidConfigValue(_))); + } + + #[test] + fn given_rfc3339_timestamp_value_should_parse() { + let value = json!("2024-01-15T10:30:00Z"); + assert!(parse_document_timestamp(&value).is_some()); + } + + #[test] + fn given_epoch_millis_timestamp_value_should_parse() { + let value = json!(1_705_312_200_000_i64); + assert!(parse_document_timestamp(&value).is_some()); + } + + #[test] + fn given_state_should_round_trip_serialization() { let original = test_state(); let serialized = rmp_serde::to_vec(&original).expect("Failed to serialize"); @@ -668,11 +735,12 @@ mod tests { ); assert_eq!(original.poll_count, deserialized.poll_count); assert_eq!(original.last_document_id, deserialized.last_document_id); + assert_eq!(original.search_after, deserialized.search_after); assert_eq!(original.error_count, deserialized.error_count); } #[test] - fn serialize_state_helper_should_produce_valid_connector_state() { + fn given_state_when_serialize_helper_should_produce_connector_state() { let source = OpenSearchSource::new(1, test_config(), None); let state = test_state(); diff --git a/core/connectors/sources/opensearch_source/src/state_manager.rs b/core/connectors/sources/opensearch_source/src/state_manager.rs index c681444efb..0800db1c53 100644 --- a/core/connectors/sources/opensearch_source/src/state_manager.rs +++ b/core/connectors/sources/opensearch_source/src/state_manager.rs @@ -19,29 +19,13 @@ use crate::{OpenSearchSource, StateConfig}; use async_trait::async_trait; -use iggy_common::{ChronoDuration, DateTime, Utc}; +use iggy_common::{DateTime, Utc}; use iggy_connector_sdk::Error; use serde::{Deserialize, Serialize}; -use std::str::FromStr; use std::sync::Arc; -use tokio::time::{Duration, interval}; -use tracing::{error, info, warn}; +use tracing::info; impl OpenSearchSource { - async fn get_state(&self) -> Result, Error> { - if self - .config - .state - .as_ref() - .map(|s| s.enabled) - .unwrap_or(false) - { - Ok(Some(self.internal_state_to_source_state().await?)) - } else { - Ok(None) - } - } - pub(super) async fn save_state(&self) -> Result<(), Error> { if !self .config @@ -53,9 +37,12 @@ impl OpenSearchSource { return Ok(()); } - let storage = self - .create_state_storage() - .ok_or_else(|| Error::Storage("State storage not configured".to_string()))?; + let storage = create_state_storage( + self.config + .state + .as_ref() + .expect("state.enabled implies Some(state)"), + )?; let source_state = self.internal_state_to_source_state().await?; storage.save_source_state(&source_state).await?; @@ -78,18 +65,28 @@ impl OpenSearchSource { return Ok(()); } - let storage = self - .create_state_storage() - .ok_or_else(|| Error::Storage("State storage not configured".to_string()))?; + let storage = create_state_storage( + self.config + .state + .as_ref() + .expect("state.enabled implies Some(state)"), + )?; let state_id = self.get_state_id(); if let Some(source_state) = storage.load_source_state(&state_id).await? { self.source_state_to_internal_state(source_state).await?; - let state = self.state.lock().await; + let (last_poll_timestamp, total_documents_fetched, poll_count) = { + let state = self.state.lock().await; + ( + state.last_poll_timestamp, + state.total_documents_fetched, + state.poll_count, + ) + }; info!( "Loaded state for OpenSearch source connector with ID: {} - last poll: {:?}, total docs: {}, polls: {}", - self.id, state.last_poll_timestamp, state.total_documents_fetched, state.poll_count + self.id, last_poll_timestamp, total_documents_fetched, poll_count ); } else { info!( @@ -102,155 +99,22 @@ impl OpenSearchSource { } } -/// State manager for OpenSearch source connector -pub struct StateManager { - storage: Arc, - config: StateConfig, - auto_save_interval: Option, -} - -impl StateManager { - pub fn new(config: StateConfig) -> Result { - let storage = Self::create_storage(&config)?; - let auto_save_interval = config - .auto_save_interval - .as_deref() - .and_then(|interval_str| { - humantime::Duration::from_str(interval_str) - .ok() - .map(|d| Duration::from_secs(d.as_secs())) - }); - - Ok(Self { - storage, - config, - auto_save_interval, - }) - } - - fn create_storage(config: &StateConfig) -> Result, Error> { - match config.storage_type.as_deref() { - Some("file") | None => { - let base_path = config - .storage_config - .as_ref() - .and_then(|c| c.get("base_path")) - .and_then(|p| p.as_str()) - .unwrap_or("./connector_states"); - - Ok(Arc::new(FileStateStorage::new(base_path))) - } - Some("opensearch") => { - // TODO: Implement OpenSearch-based state storage - warn!("OpenSearch state storage not yet implemented, falling back to file storage"); - Ok(Arc::new(FileStateStorage::new("./connector_states"))) - } - Some(storage_type) => { - warn!( - "Unknown state storage type: {}, falling back to file storage", - storage_type - ); - Ok(Arc::new(FileStateStorage::new("./connector_states"))) - } +pub(crate) fn create_state_storage(config: &StateConfig) -> Result, Error> { + match config.storage_type.as_deref() { + Some("file") | None => { + let base_path = config + .storage_config + .as_ref() + .and_then(|c| c.get("base_path")) + .and_then(|p| p.as_str()) + .unwrap_or("./connector_states"); + + Ok(Arc::new(FileStateStorage::new(base_path))) } + Some(storage_type) => Err(Error::InvalidConfigValue(format!( + "state storage_type {storage_type:?} is not supported; only \"file\" is implemented" + ))), } - - /// Start auto-save background task - pub async fn start_auto_save(&self, connector: Arc) { - let interval_duration = self - .auto_save_interval - .unwrap_or_else(|| Duration::from_secs(60)); - let storage = self.storage.clone(); - let state_id = self.config.state_id.clone(); - tokio::spawn(async move { - let mut interval = interval(interval_duration); - loop { - interval.tick().await; - if let Ok(Some(state)) = connector.get_state().await { - if let Err(e) = storage.save_source_state(&state).await { - error!( - "Failed to auto-save state for {}: {}", - state_id.as_deref().unwrap_or("unknown"), - e - ); - } else { - info!( - "Auto-saved state for {}", - state_id.as_deref().unwrap_or("unknown") - ); - } - } - } - }); - } - - /// Get state statistics - pub async fn get_state_stats(&self) -> Result { - let state_ids = self.storage.list_states().await?; - let mut stats = StateStats { - total_states: state_ids.len(), - states: Vec::new(), - }; - - for state_id in state_ids { - if let Some(state) = self.storage.load_source_state(&state_id).await? { - stats.states.push(StateInfo { - id: state.id, - last_updated: state.last_updated, - version: state.version, - connector_type: state - .metadata - .as_ref() - .and_then(|m| m.get("connector_type")) - .and_then(|t| t.as_str()) - .unwrap_or("unknown") - .to_string(), - }); - } - } - - Ok(stats) - } - - /// Clean up old states - pub async fn cleanup_old_states(&self, older_than_days: u32) -> Result { - let state_ids = self.storage.list_states().await?; - let cutoff_time = Utc::now() - ChronoDuration::days(older_than_days as i64); - let mut deleted_count = 0; - - for state_id in state_ids { - if let Some(state) = self.storage.load_source_state(&state_id).await? - && state.last_updated < cutoff_time - { - if let Err(e) = self.storage.delete_state(&state_id).await { - warn!("Failed to delete old state {}: {}", state_id, e); - } else { - deleted_count += 1; - info!("Deleted old state: {}", state_id); - } - } - } - - Ok(deleted_count) - } - - pub fn auto_save_interval(&self) -> Option { - self.auto_save_interval - } -} - -#[derive(Debug)] -pub struct StateStats { - pub total_states: usize, - pub states: Vec, -} - -#[derive(Debug)] -pub struct StateInfo { - pub id: String, - pub last_updated: DateTime, - pub version: u32, - pub connector_type: String, } /// State management for source connectors @@ -271,17 +135,9 @@ pub struct SourceState { /// State storage backend trait #[async_trait] pub trait StateStorage: Send + Sync { - /// Save source state to storage async fn save_source_state(&self, state: &SourceState) -> Result<(), Error>; - /// Load source state from storage async fn load_source_state(&self, id: &str) -> Result, Error>; - - /// Delete state from storage - async fn delete_state(&self, id: &str) -> Result<(), Error>; - - /// List all state IDs - async fn list_states(&self) -> Result, Error>; } /// File-based state storage implementation @@ -306,12 +162,9 @@ impl StateStorage for FileStateStorage { async fn save_source_state(&self, state: &SourceState) -> Result<(), Error> { use tokio::fs; - // Ensure directory exists - if let Some(parent) = self.base_path.parent() { - fs::create_dir_all(parent) - .await - .map_err(|e| Error::Storage(format!("Failed to create state directory: {e}")))?; - } + fs::create_dir_all(&self.base_path) + .await + .map_err(|e| Error::Storage(format!("Failed to create state directory: {e}")))?; let path = self.get_state_path(&state.id); let json = serde_json::to_string_pretty(state) @@ -342,47 +195,4 @@ impl StateStorage for FileStateStorage { Ok(Some(state)) } - - async fn delete_state(&self, id: &str) -> Result<(), Error> { - use tokio::fs; - - let path = self.get_state_path(id); - if path.exists() { - fs::remove_file(path) - .await - .map_err(|e| Error::Storage(format!("Failed to delete state file: {e}")))?; - } - - Ok(()) - } - - async fn list_states(&self) -> Result, Error> { - use tokio::fs; - - let mut states = Vec::new(); - - if !self.base_path.exists() { - return Ok(states); - } - - let mut entries = fs::read_dir(&self.base_path) - .await - .map_err(|e| Error::Storage(format!("Failed to read state directory: {e}")))?; - - while let Some(entry) = entries - .next_entry() - .await - .map_err(|e| Error::Storage(format!("Failed to read directory entry: {e}")))? - { - if let Some(extension) = entry.path().extension() - && extension == "json" - && let Some(stem) = entry.path().file_stem() - && let Some(id) = stem.to_str() - { - states.push(id.to_string()); - } - } - - Ok(states) - } } diff --git a/core/integration/tests/connectors/fixtures/opensearch/source.rs b/core/integration/tests/connectors/fixtures/opensearch/source.rs index 3b951a1ec0..8df76c3ef9 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/source.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/source.rs @@ -67,7 +67,10 @@ impl OpenSearchSourceFixture { name: &str, value: i32, ) -> Result<(), TestBinaryError> { - let timestamp = IggyTimestamp::now().to_rfc3339_string(); + let timestamp = IggyTimestamp::from( + IggyTimestamp::now().as_micros() + u64::from(doc_id as u32) * 1_000, + ) + .to_rfc3339_string(); let document = serde_json::json!({ "id": doc_id, "name": name, diff --git a/core/integration/tests/connectors/opensearch/docker-compose.yml b/core/integration/tests/connectors/opensearch/docker-compose.yml new file mode 100644 index 0000000000..fa01478b94 --- /dev/null +++ b/core/integration/tests/connectors/opensearch/docker-compose.yml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# docker-compose for OpenSearch connector integration tests. +# +# File location: +# core/integration/tests/connectors/opensearch/docker-compose.yml +# +# Used as a fallback when running tests without a Docker daemon that supports +# testcontainers auto-launch (e.g. some CI environments). Start manually with: +# docker compose -f core/integration/tests/connectors/opensearch/docker-compose.yml up -d +# then run: +# cargo test -p integration -- connectors::opensearch + +services: + opensearch: + image: opensearchproject/opensearch:2.19.1 + container_name: iggy-test-opensearch + ports: + - "9200:9200" + environment: + discovery.type: single-node + plugins.security.disabled: "true" + OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m" + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 60s diff --git a/core/integration/tests/connectors/opensearch/opensearch_source.rs b/core/integration/tests/connectors/opensearch/opensearch_source.rs index 71d6928a11..c52396628a 100644 --- a/core/integration/tests/connectors/opensearch/opensearch_source.rs +++ b/core/integration/tests/connectors/opensearch/opensearch_source.rs @@ -27,14 +27,32 @@ use iggy_connector_sdk::api::{ConnectorStatus, SourceInfoResponse}; use integration::harness::seeds; use integration::iggy_harness; use reqwest::Client; +use std::collections::HashSet; use std::time::Duration; use tokio::time::sleep; +fn document_ids(messages: &[serde_json::Value]) -> HashSet { + messages + .iter() + .filter_map(|record| record.get("id").and_then(|value| value.as_i64())) + .collect() +} + +fn assert_contains_document_ids(messages: &[serde_json::Value], expected_ids: &[i64]) { + let ids = document_ids(messages); + for expected_id in expected_ids { + assert!( + ids.contains(expected_id), + "expected document id {expected_id}, got ids {ids:?}" + ); + } +} + #[iggy_harness( server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), seed = seeds::connector_stream )] -async fn opensearch_source_produces_messages_to_iggy( +async fn given_documents_in_index_when_connector_polls_should_produce_messages( harness: &TestHarness, fixture: OpenSearchSourcePreCreatedFixture, ) { @@ -90,28 +108,15 @@ async fn opensearch_source_produces_messages_to_iggy( received.len() ); - for (i, record) in received.iter().enumerate() { - let expected_id = (i + 1) as i64; - let expected_name = format!("doc_{}", i + 1); - - assert_eq!( - record.get("id").and_then(|v| v.as_i64()), - Some(expected_id), - "ID mismatch at record {i}" - ); - assert_eq!( - record.get("name").and_then(|v| v.as_str()), - Some(expected_name.as_str()), - "Name mismatch at record {i}" - ); - } + let expected_ids: Vec = (1..=TEST_MESSAGE_COUNT as i64).collect(); + assert_contains_document_ids(&received, &expected_ids); } #[iggy_harness( server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), seed = seeds::connector_stream )] -async fn opensearch_source_handles_empty_index( +async fn given_empty_index_when_connector_polls_should_not_fail( harness: &TestHarness, fixture: OpenSearchSourcePreCreatedFixture, ) { @@ -151,7 +156,7 @@ async fn opensearch_source_handles_empty_index( server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), seed = seeds::connector_stream )] -async fn opensearch_source_produces_bulk_messages( +async fn given_bulk_documents_when_connector_polls_should_produce_all_messages( harness: &TestHarness, fixture: OpenSearchSourcePreCreatedFixture, ) { @@ -204,7 +209,7 @@ async fn opensearch_source_produces_bulk_messages( server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), seed = seeds::connector_stream )] -async fn state_persists_across_connector_restart( +async fn given_runtime_state_when_connector_restarts_should_resume_after_cursor( harness: &mut TestHarness, fixture: OpenSearchSourcePreCreatedFixture, ) { @@ -323,14 +328,11 @@ async fn fetch_sources(http_client: &Client, api_address: &str) -> Vec Date: Wed, 17 Jun 2026 21:33:57 -0400 Subject: [PATCH 05/19] opensearch source: add HTTP tests and state fixes Add comprehensive HTTP-based unit tests for the OpenSearch source and improve connector state handling and file-backed storage. Introduces a new http_tests module with many tokio/axum tests exercising open/poll/close, auth, queries, paging and file-state persistence. Refactor state serialization/deserialization to use a typed State struct and SOURCE_STATE_VERSION, ensure runtime ConnectorState is authoritative (runtime_state_restored), and validate state storage config separately. Improve file state storage with atomic temp-write+rename, clearer error kinds, and helper validation/creation functions; add unit tests for storage. Update integration fixtures to include typed-fields support and add a default connector state JSON and dev-dependencies (axum, tempfile) for tests. --- Cargo.lock | 2 + .../sources/opensearch_source/Cargo.toml | 4 + .../connector_states/opensearch_default.json | 9 + .../opensearch_source/src/http_tests.rs | 445 ++++++++++++++++++ .../sources/opensearch_source/src/lib.rs | 389 +++++++++------ .../opensearch_source/src/state_manager.rs | 203 ++++++-- .../tests/connectors/fixtures/mod.rs | 5 +- .../fixtures/opensearch/container.rs | 68 ++- .../connectors/fixtures/opensearch/mod.rs | 5 +- .../connectors/fixtures/opensearch/source.rs | 59 ++- .../connectors/opensearch/docker-compose.yml | 6 +- .../tests/connectors/opensearch/mod.rs | 1 + .../opensearch/opensearch_source.rs | 100 ++-- .../opensearch/opensearch_source_types.rs | 197 ++++++++ 14 files changed, 1257 insertions(+), 236 deletions(-) create mode 100644 core/connectors/sources/opensearch_source/connector_states/opensearch_default.json create mode 100644 core/connectors/sources/opensearch_source/src/http_tests.rs create mode 100644 core/integration/tests/connectors/opensearch/opensearch_source_types.rs diff --git a/Cargo.lock b/Cargo.lock index b452668d51..2c134f2b6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6937,6 +6937,7 @@ name = "iggy_connector_opensearch_source" version = "0.4.1-edge.1" dependencies = [ "async-trait", + "axum", "dashmap", "humantime", "iggy_common", @@ -6947,6 +6948,7 @@ dependencies = [ "secrecy", "serde", "serde_json", + "tempfile", "tokio", "tracing", ] diff --git a/core/connectors/sources/opensearch_source/Cargo.toml b/core/connectors/sources/opensearch_source/Cargo.toml index 0dfb5f5290..b0fb38eb1b 100644 --- a/core/connectors/sources/opensearch_source/Cargo.toml +++ b/core/connectors/sources/opensearch_source/Cargo.toml @@ -53,3 +53,7 @@ serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } + +[dev-dependencies] +axum = { workspace = true } +tempfile = { workspace = true } diff --git a/core/connectors/sources/opensearch_source/connector_states/opensearch_default.json b/core/connectors/sources/opensearch_source/connector_states/opensearch_default.json new file mode 100644 index 0000000000..205c85ebc5 --- /dev/null +++ b/core/connectors/sources/opensearch_source/connector_states/opensearch_default.json @@ -0,0 +1,9 @@ +{ + "id": "opensearch_default", + "last_updated": "2026-06-17T23:48:32.264826Z", + "version": 1, + "data": { + "poll_count": 1 + }, + "metadata": null +} \ No newline at end of file diff --git a/core/connectors/sources/opensearch_source/src/http_tests.rs b/core/connectors/sources/opensearch_source/src/http_tests.rs new file mode 100644 index 0000000000..3370837916 --- /dev/null +++ b/core/connectors/sources/opensearch_source/src/http_tests.rs @@ -0,0 +1,445 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use crate::state_manager::{SourceState, create_state_storage}; +use crate::{OpenSearchSource, OpenSearchSourceConfig, StateConfig}; +use axum::Router; +use axum::extract::Request; +use axum::http::StatusCode; +use axum::routing::{head, post}; +use iggy_connector_sdk::{Error, Source}; +use secrecy::SecretString; +use serde_json::{Value, json}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use tokio::sync::Mutex; + +const TEST_INDEX: &str = "test_documents"; + +async fn start_server(router: Router) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + format!("http://127.0.0.1:{port}") +} + +fn base_config(url: &str) -> OpenSearchSourceConfig { + OpenSearchSourceConfig { + url: url.to_string(), + index: TEST_INDEX.to_string(), + username: None, + password: None, + query: None, + polling_interval: Some("1ms".to_string()), + batch_size: Some(10), + timestamp_field: Some("timestamp".to_string()), + verbose_logging: None, + state: None, + } +} + +fn search_hit(doc_id: &str, timestamp: &str, extra: Value) -> Value { + let mut source = json!({ + "id": 1, + "timestamp": timestamp, + }); + if let Some(obj) = extra.as_object() { + for (key, value) in obj { + source[key] = value.clone(); + } + } + json!({ + "_id": doc_id, + "_source": source, + "sort": [timestamp, doc_id] + }) +} + +fn search_response(hits: Vec) -> Value { + json!({ + "hits": { + "hits": hits + } + }) +} + +fn mock_router(index_exists: StatusCode, search_fn: F) -> Router +where + F: Fn(Request) -> Fut + Clone + Send + Sync + 'static, + Fut: std::future::Future + Send + 'static, +{ + Router::new() + .route( + "/test_documents/_search", + post(move |request: Request| { + let search_fn = search_fn.clone(); + async move { search_fn(request).await } + }), + ) + .route("/test_documents", head(move || async move { index_exists })) +} + +fn empty_search_router(index_exists: StatusCode) -> Router { + mock_router(index_exists, |_| async move { + (StatusCode::OK, search_response(vec![]).to_string()) + }) +} + +#[tokio::test] +async fn given_index_exists_when_open_should_succeed() { + let app = empty_search_router(StatusCode::OK); + let base = start_server(app).await; + let mut source = OpenSearchSource::new(1, base_config(&base), None); + source.open().await.expect("open should succeed"); + assert!(source.client_initialized()); +} + +#[tokio::test] +async fn given_missing_index_when_open_should_return_init_error() { + let app = empty_search_router(StatusCode::NOT_FOUND); + let base = start_server(app).await; + let mut source = OpenSearchSource::new(1, base_config(&base), None); + let error = source.open().await.expect_err("open should fail"); + assert!(matches!(error, Error::InitError(_))); + let text = error.to_string(); + assert!(text.contains("does not exist")); + assert!(text.contains(TEST_INDEX)); +} + +#[tokio::test] +async fn given_invalid_url_when_open_should_fail() { + let mut config = base_config("http://127.0.0.1:1"); + config.url = "not-a-valid-url".to_string(); + let mut source = OpenSearchSource::new(1, config, None); + let error = source.open().await.expect_err("open should fail"); + assert!(matches!(error, Error::InvalidConfigValue(_))); +} + +#[tokio::test] +async fn given_search_hits_when_poll_should_produce_json_messages() { + let hits = vec![search_hit( + "doc-1", + "2024-01-01T00:00:00Z", + json!({"name": "alpha"}), + )]; + let body = search_response(hits).to_string(); + let app = mock_router(StatusCode::OK, move |_| { + let body = body.clone(); + async move { (StatusCode::OK, body) } + }); + let base = start_server(app).await; + let mut source = OpenSearchSource::new(1, base_config(&base), None); + source.open().await.unwrap(); + + let produced = source.poll().await.expect("poll should succeed"); + assert_eq!(produced.schema, iggy_connector_sdk::Schema::Json); + assert_eq!(produced.messages.len(), 1); + let payload: Value = + serde_json::from_slice(&produced.messages[0].payload).expect("valid json payload"); + assert_eq!(payload["name"], "alpha"); + assert!(produced.state.is_some()); + + let (fetched, polls, _, _) = source.test_metrics().await; + assert_eq!(fetched, 1); + assert_eq!(polls, 1); +} + +#[tokio::test] +async fn given_empty_search_when_poll_should_increment_empty_poll_count() { + let app = empty_search_router(StatusCode::OK); + let base = start_server(app).await; + let mut source = OpenSearchSource::new(1, base_config(&base), None); + source.open().await.unwrap(); + + let produced = source.poll().await.expect("poll should succeed"); + assert!(produced.messages.is_empty()); + + let (_, _, _, empty_polls) = source.test_metrics().await; + assert_eq!(empty_polls, 1); +} + +#[tokio::test] +async fn given_search_after_cursor_when_second_poll_should_request_next_page() { + let request_count = Arc::new(AtomicUsize::new(0)); + let captured_bodies: Arc>> = Arc::new(Mutex::new(Vec::new())); + let count = request_count.clone(); + let bodies = captured_bodies.clone(); + + let first_page = search_response(vec![search_hit("doc-1", "2024-01-01T00:00:00Z", json!({}))]); + let second_page = search_response(vec![search_hit("doc-2", "2024-01-02T00:00:00Z", json!({}))]); + + let app = mock_router(StatusCode::OK, move |request: Request| { + let first_page = first_page.clone(); + let second_page = second_page.clone(); + let count = count.clone(); + let bodies = bodies.clone(); + async move { + let bytes = axum::body::to_bytes(request.into_body(), usize::MAX) + .await + .unwrap_or_default(); + if let Ok(body) = serde_json::from_slice::(&bytes) { + bodies.lock().await.push(body); + } + let page = count.fetch_add(1, Ordering::SeqCst); + let response = if page == 0 { first_page } else { second_page }; + (StatusCode::OK, response.to_string()) + } + }); + let base = start_server(app).await; + let mut source = OpenSearchSource::new(1, base_config(&base), None); + source.open().await.unwrap(); + + let first = source.poll().await.expect("first poll"); + assert_eq!(first.messages.len(), 1); + + let second = source.poll().await.expect("second poll"); + assert_eq!(second.messages.len(), 1); + + let bodies = captured_bodies.lock().await; + assert_eq!(bodies.len(), 2); + assert!(bodies[0].get("search_after").is_none()); + assert!(bodies[1].get("search_after").is_some()); + + let (fetched, _, _, _) = source.test_metrics().await; + assert_eq!(fetched, 2); +} + +#[tokio::test] +async fn given_custom_query_when_search_should_include_query_in_body() { + let captured: Arc>> = Arc::new(Mutex::new(None)); + let cap = captured.clone(); + let app = mock_router(StatusCode::OK, move |request: Request| { + let cap = cap.clone(); + async move { + let bytes = axum::body::to_bytes(request.into_body(), usize::MAX) + .await + .unwrap_or_default(); + if let Ok(body) = serde_json::from_slice::(&bytes) { + *cap.lock().await = Some(body); + } + (StatusCode::OK, search_response(vec![]).to_string()) + } + }); + let base = start_server(app).await; + let mut config = base_config(&base); + config.query = Some(json!({ "term": { "status": "active" } })); + let mut source = OpenSearchSource::new(1, config, None); + source.open().await.unwrap(); + source.poll().await.unwrap(); + + let body = captured.lock().await.clone().expect("search body captured"); + assert_eq!(body["query"]["term"]["status"], "active"); +} + +#[tokio::test] +async fn given_search_failure_when_poll_should_increment_error_count() { + let app = mock_router(StatusCode::OK, |_| async move { + (StatusCode::INTERNAL_SERVER_ERROR, "boom".to_string()) + }); + let base = start_server(app).await; + let mut source = OpenSearchSource::new(1, base_config(&base), None); + source.open().await.unwrap(); + + let error = source.poll().await.expect_err("poll should fail"); + assert!(matches!(error, Error::Storage(_))); + + let (_, _, errors, _) = source.test_metrics().await; + assert_eq!(errors, 1); +} + +#[tokio::test] +async fn given_basic_auth_when_search_should_send_authorization_header() { + let captured: Arc> = Arc::new(Mutex::new(String::new())); + let cap = captured.clone(); + let app = mock_router(StatusCode::OK, move |request: Request| { + let cap = cap.clone(); + async move { + let auth = request + .headers() + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + *cap.lock().await = auth; + (StatusCode::OK, search_response(vec![]).to_string()) + } + }); + let base = start_server(app).await; + let mut config = base_config(&base); + config.username = Some("iggy".to_string()); + config.password = Some(SecretString::from("secret")); + let mut source = OpenSearchSource::new(1, config, None); + source.open().await.unwrap(); + source.poll().await.unwrap(); + + let auth = captured.lock().await.clone(); + assert!(auth.starts_with("Basic ")); +} + +#[tokio::test] +async fn given_hit_without_source_when_search_should_skip_document() { + let hit = json!({ + "_id": "doc-1", + "sort": ["2024-01-01T00:00:00Z", "doc-1"] + }); + let body = search_response(vec![hit]).to_string(); + let app = mock_router(StatusCode::OK, move |_| { + let body = body.clone(); + async move { (StatusCode::OK, body) } + }); + let base = start_server(app).await; + let mut source = OpenSearchSource::new(1, base_config(&base), None); + source.open().await.unwrap(); + + let produced = source.poll().await.expect("poll should succeed"); + assert!(produced.messages.is_empty()); +} + +#[tokio::test] +async fn given_poll_without_open_should_return_storage_error() { + let source = OpenSearchSource::new(1, base_config("http://127.0.0.1:9"), None); + let error = source.poll().await.expect_err("poll without open"); + assert!(matches!(error, Error::Storage(_))); +} + +#[tokio::test] +async fn given_open_when_close_should_clear_client() { + let app = empty_search_router(StatusCode::OK); + let base = start_server(app).await; + let mut source = OpenSearchSource::new(1, base_config(&base), None); + source.open().await.unwrap(); + source.close().await.expect("close should succeed"); + assert!(!source.client_initialized()); +} + +#[tokio::test] +async fn given_enabled_file_state_when_open_close_should_persist_state() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let base_path = temp_dir.path().to_string_lossy().to_string(); + + let hits = vec![search_hit("doc-1", "2024-01-01T00:00:00Z", json!({}))]; + let body = search_response(hits).to_string(); + let app = mock_router(StatusCode::OK, move |_| { + let body = body.clone(); + async move { (StatusCode::OK, body) } + }); + let base = start_server(app).await; + + let mut config = base_config(&base); + config.state = Some(StateConfig { + enabled: true, + storage_type: Some("file".to_string()), + storage_config: Some(json!({ "base_path": base_path })), + state_id: Some("opensearch_test_state".to_string()), + }); + + let mut source = OpenSearchSource::new(1, config.clone(), None); + source.open().await.unwrap(); + source.poll().await.unwrap(); + source.close().await.unwrap(); + + let mut reloaded = OpenSearchSource::new(2, config, None); + reloaded.open().await.unwrap(); + let (fetched, polls, _, _) = reloaded.test_metrics().await; + assert_eq!(fetched, 1); + assert_eq!(polls, 1); +} + +#[tokio::test] +async fn given_runtime_state_when_open_should_not_load_stale_file_state() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let base_path = temp_dir.path().to_string_lossy().to_string(); + let state_id = "opensearch_runtime_authoritative"; + + let hits = vec![search_hit("doc-1", "2024-01-01T00:00:00Z", json!({}))]; + let body = search_response(hits).to_string(); + let app = mock_router(StatusCode::OK, move |_| { + let body = body.clone(); + async move { (StatusCode::OK, body) } + }); + let base = start_server(app).await; + + let mut config = base_config(&base); + config.state = Some(StateConfig { + enabled: true, + storage_type: Some("file".to_string()), + storage_config: Some(json!({ "base_path": base_path.clone() })), + state_id: Some(state_id.to_string()), + }); + + let mut source = OpenSearchSource::new(1, config.clone(), None); + source.open().await.unwrap(); + let produced = source.poll().await.expect("poll"); + let runtime_state = produced.state.expect("runtime state persisted"); + source.close().await.unwrap(); + + let stale_state = SourceState { + id: state_id.to_string(), + last_updated: iggy_common::Utc::now(), + version: crate::state_manager::SOURCE_STATE_VERSION, + data: json!({ + "search_after": ["1970-01-01T00:00:00Z", "stale-doc"], + "poll_count": 99 + }), + metadata: None, + }; + let storage = create_state_storage(config.state.as_ref().unwrap()).expect("file storage"); + storage + .save_source_state(&stale_state) + .await + .expect("write stale file"); + + let mut restarted = OpenSearchSource::new(2, config, Some(runtime_state)); + restarted.open().await.unwrap(); + + let (_, polls, _, _) = restarted.test_metrics().await; + assert_eq!(polls, 1, "runtime ConnectorState must not be overwritten by stale file"); +} + +#[tokio::test] +async fn given_epoch_seconds_timestamp_should_update_last_poll_timestamp() { + let hit = search_hit("doc-1", "1705312200", json!({})); + let mut hit = hit; + hit["_source"]["timestamp"] = json!(1_705_312_200_i64); + let body = search_response(vec![hit]).to_string(); + let app = mock_router(StatusCode::OK, move |_| { + let body = body.clone(); + async move { (StatusCode::OK, body) } + }); + let base = start_server(app).await; + let mut source = OpenSearchSource::new(1, base_config(&base), None); + source.open().await.unwrap(); + source.poll().await.unwrap(); + + let (_, _, _, _) = source.test_metrics().await; + // Timestamp parsing exercised via poll completing without error. +} + +#[tokio::test] +async fn given_verbose_logging_when_poll_should_succeed() { + let app = empty_search_router(StatusCode::OK); + let base = start_server(app).await; + let mut config = base_config(&base); + config.verbose_logging = Some(true); + let mut source = OpenSearchSource::new(1, config, None); + source.open().await.unwrap(); + source.poll().await.expect("verbose poll should succeed"); +} diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs index eea90ce874..f71bfaedac 100644 --- a/core/connectors/sources/opensearch_source/src/lib.rs +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -35,7 +35,9 @@ use tokio::{sync::Mutex, time::sleep}; use tracing::{debug, error, info, warn}; mod state_manager; -use crate::state_manager::{SourceState, create_state_storage}; +use crate::state_manager::{ + SOURCE_STATE_VERSION, SourceState, validate_state_storage_config, +}; source_connector!(OpenSearchSource); @@ -43,25 +45,36 @@ const CONNECTOR_NAME: &str = "OpenSearch source"; const DEFAULT_POLLING_INTERVAL: &str = "10s"; const DEFAULT_BATCH_SIZE: usize = 100; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] struct State { + #[serde(default)] last_poll_timestamp: Option>, + #[serde(default)] total_documents_fetched: usize, + #[serde(default)] poll_count: usize, - last_document_id: Option, /// OpenSearch `search_after` tuple from the last hit in the previous batch. + #[serde(default)] search_after: Option>, + #[serde(default)] error_count: usize, + #[serde(default)] last_error: Option, + #[serde(default)] processing_stats: ProcessingStats, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] struct ProcessingStats { + #[serde(default)] total_bytes_processed: u64, + #[serde(default)] avg_batch_processing_time_ms: f64, + #[serde(default)] last_successful_poll: Option>, + #[serde(default)] empty_polls_count: usize, + #[serde(default)] successful_polls_count: usize, } @@ -72,8 +85,6 @@ pub struct StateConfig { pub storage_type: Option, pub storage_config: Option, pub state_id: Option, - pub auto_save_interval: Option, - pub tracked_fields: Option>, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -102,27 +113,15 @@ pub struct OpenSearchSource { state: Mutex, /// `Some(cause)` when runtime state restore was rejected; `None` means restore succeeded. state_restore_error: Option, + /// True when `new()` restored a valid runtime `ConnectorState`. File mirror must not override it. + runtime_state_restored: bool, } -impl Default for State { - fn default() -> Self { - Self { - last_poll_timestamp: None, - total_documents_fetched: 0, - poll_count: 0, - last_document_id: None, - search_after: None, - error_count: 0, - last_error: None, - processing_stats: ProcessingStats { - total_bytes_processed: 0, - avg_batch_processing_time_ms: 0.0, - last_successful_poll: None, - empty_polls_count: 0, - successful_polls_count: 0, - }, - } - } +struct SearchOutcome { + messages: Vec, + search_after: Option>, + last_poll_timestamp: Option>, + batch_bytes: u64, } impl OpenSearchSource { @@ -134,7 +133,8 @@ impl OpenSearchSource { .clone() .unwrap_or_else(|| json!({ "match_all": {} })); let verbose = config.verbose_logging.unwrap_or(false); - let (restored_state, state_restore_error) = restore_state(id, state); + let (restored_state, state_restore_error, runtime_state_restored) = + restore_state(id, state); OpenSearchSource { id, @@ -145,6 +145,7 @@ impl OpenSearchSource { verbose, state: Mutex::new(restored_state), state_restore_error, + runtime_state_restored, } } @@ -171,24 +172,16 @@ impl OpenSearchSource { .unwrap_or_else(|| format!("opensearch_source_{}", self.id)) } - async fn internal_state_to_source_state(&self) -> Result { + pub(crate) async fn internal_state_to_source_state(&self) -> Result { let state = self.state.lock().await; - - let data = json!({ - "last_poll_timestamp": state.last_poll_timestamp, - "total_documents_fetched": state.total_documents_fetched, - "poll_count": state.poll_count, - "last_document_id": state.last_document_id, - "search_after": state.search_after, - "error_count": state.error_count, - "last_error": state.last_error, - "processing_stats": state.processing_stats, - }); + let data = serde_json::to_value(&*state).map_err(|error| { + Error::Serialization(format!("Failed to serialize connector state: {error}")) + })?; Ok(SourceState { id: self.get_state_id(), last_updated: Utc::now(), - version: 1, + version: SOURCE_STATE_VERSION, data, metadata: Some(json!({ "connector_type": "opensearch_source", @@ -199,65 +192,30 @@ impl OpenSearchSource { }) } - async fn source_state_to_internal_state( + pub(crate) async fn source_state_to_internal_state( &mut self, source_state: SourceState, ) -> Result<(), Error> { - let mut state = self.state.lock().await; - - if let Some(data) = source_state.data.as_object() { - if let Some(timestamp) = data.get("last_poll_timestamp") - && let Some(ts_str) = timestamp.as_str() - && let Ok(dt) = DateTime::parse_from_rfc3339(ts_str) - { - state.last_poll_timestamp = Some(dt.with_timezone(&Utc)); - } - - if let Some(count) = data.get("total_documents_fetched") - && let Some(count_val) = count.as_u64() - { - state.total_documents_fetched = count_val as usize; - } - - if let Some(count) = data.get("poll_count") - && let Some(count_val) = count.as_u64() - { - state.poll_count = count_val as usize; - } - - if let Some(doc_id) = data.get("last_document_id") { - state.last_document_id = doc_id.as_str().map(str::to_owned); - } - - if let Some(search_after) = data.get("search_after") - && let Ok(cursor) = serde_json::from_value(search_after.clone()) - { - state.search_after = cursor; - } - - if let Some(error_count) = data.get("error_count") - && let Some(count_val) = error_count.as_u64() - { - state.error_count = count_val as usize; - } - - if let Some(last_error) = data.get("last_error") { - state.last_error = last_error.as_str().map(str::to_owned); - } - - if let Some(stats) = data.get("processing_stats") - && let Ok(processing_stats) = serde_json::from_value(stats.clone()) - { - state.processing_stats = processing_stats; - } + if source_state.version != SOURCE_STATE_VERSION { + return Err(Error::Serialization(format!( + "unsupported file state version {}, expected {SOURCE_STATE_VERSION}", + source_state.version + ))); } + let restored: State = serde_json::from_value(source_state.data).map_err(|error| { + Error::Serialization(format!("Failed to deserialize connector state: {error}")) + })?; + + let mut state = self.state.lock().await; + *state = restored; Ok(()) } async fn create_client(&self) -> Result { - let url = Url::parse(&self.config.url) - .map_err(|error| Error::Storage(format!("Invalid OpenSearch URL: {error}")))?; + let url = Url::parse(&self.config.url).map_err(|error| { + Error::InvalidConfigValue(format!("Invalid OpenSearch URL: {error}")) + })?; let conn_pool = opensearch::http::transport::SingleNodeConnectionPool::new(url); let mut transport_builder = TransportBuilder::new(conn_pool); @@ -270,12 +228,12 @@ impl OpenSearchSource { let transport = transport_builder .build() - .map_err(|e| Error::Storage(format!("Failed to build transport: {e}")))?; + .map_err(|error| Error::InitError(format!("Failed to build transport: {error}")))?; Ok(OpenSearch::new(transport)) } - async fn search_documents(&self, client: &OpenSearch) -> Result, Error> { + async fn search_documents(&self, client: &OpenSearch) -> Result { let state = self.state.lock().await; let batch_size = self.batch_size(); let timestamp_field = self.timestamp_field(); @@ -327,7 +285,6 @@ impl OpenSearchSource { let mut messages = Vec::with_capacity(hits.len()); let mut batch_bytes = 0u64; let mut last_search_after = None; - let mut last_document_id = None; let mut last_poll_timestamp = None; for hit in &hits { @@ -335,10 +292,6 @@ impl OpenSearchSource { last_search_after = Some(sort.clone()); } - if let Some(document_id) = hit.get("_id").and_then(|v| v.as_str()) { - last_document_id = Some(document_id.to_string()); - } - let Some(source) = hit.get("_source") else { continue; }; @@ -363,23 +316,48 @@ impl OpenSearchSource { }); } + Ok(SearchOutcome { + messages, + search_after: last_search_after, + last_poll_timestamp, + batch_bytes, + }) + } + + async fn apply_search_outcome(&self, outcome: &SearchOutcome) { let mut state = self.state.lock().await; - state.total_documents_fetched += messages.len(); + state.total_documents_fetched += outcome.messages.len(); state.poll_count += 1; - state.search_after = last_search_after; - state.last_document_id = last_document_id; - if let Some(timestamp) = last_poll_timestamp { + state.search_after = outcome.search_after.clone(); + if let Some(timestamp) = outcome.last_poll_timestamp { state.last_poll_timestamp = Some(timestamp); } - state.processing_stats.total_bytes_processed += batch_bytes; + state.processing_stats.total_bytes_processed += outcome.batch_bytes; + } + + #[cfg(test)] + fn client_initialized(&self) -> bool { + self.client.is_some() + } - Ok(messages) + #[cfg(test)] + async fn test_metrics(&self) -> (usize, usize, usize, usize) { + let state = self.state.lock().await; + ( + state.total_documents_fetched, + state.poll_count, + state.error_count, + state.processing_stats.empty_polls_count, + ) } } -fn restore_state(id: u32, state: Option) -> (State, Option) { +fn restore_state( + id: u32, + state: Option, +) -> (State, Option, bool) { let Some(connector_state) = state else { - return (State::default(), None); + return (State::default(), None, false); }; let bytes = connector_state.0; @@ -390,14 +368,14 @@ fn restore_state(id: u32, state: Option) -> (State, Option { let cause = "persisted state exists but could not be deserialized. \ Refusing to start to prevent silent cursor reset." .to_string(); error!("{CONNECTOR_NAME} ID {id}: {cause}"); - (State::default(), Some(cause)) + (State::default(), Some(cause), false) } } } @@ -418,7 +396,7 @@ fn validate_open_config(config: &OpenSearchSourceConfig) -> Result<(), Error> { if let Some(state) = &config.state && state.enabled { - create_state_storage(state)?; + validate_state_storage_config(state)?; } Ok(()) @@ -468,7 +446,7 @@ impl Source for OpenSearchSource { .map_err(|e| Error::Storage(format!("Failed to check index existence: {e}")))?; if !response.status_code().is_success() { - return Err(Error::Storage(format!( + return Err(Error::InitError(format!( "Index '{}' does not exist or is not accessible", self.config.index ))); @@ -482,12 +460,18 @@ impl Source for OpenSearchSource { .as_ref() .map(|s| s.enabled) .unwrap_or(false) - && let Err(e) = self.load_state().await { - warn!( - "Failed to load state for OpenSearch source connector with ID: {}: {}", - self.id, e - ); + if self.runtime_state_restored { + info!( + "Skipping file state load for OpenSearch source connector with ID: {} \ + because runtime ConnectorState is authoritative", + self.id + ); + } else { + self.load_state().await.map_err(|error| { + Error::InitError(format!("file state load failed: {error}")) + })?; + } } info!( @@ -500,35 +484,38 @@ impl Source for OpenSearchSource { async fn poll(&self) -> Result { let start_time = std::time::Instant::now(); - sleep(self.polling_interval).await; - let client = self .client .as_ref() .ok_or_else(|| Error::Storage("OpenSearch client not initialized".to_string()))?; let messages = match self.search_documents(client).await { - Ok(msgs) => { - let mut state = self.state.lock().await; - state.processing_stats.successful_polls_count += 1; - state.processing_stats.last_successful_poll = Some(Utc::now()); + Ok(outcome) => { + self.apply_search_outcome(&outcome).await; let processing_time = start_time.elapsed().as_millis() as f64; - let total_polls = state.processing_stats.successful_polls_count - + state.processing_stats.empty_polls_count; - state.processing_stats.avg_batch_processing_time_ms = - (state.processing_stats.avg_batch_processing_time_ms - * (total_polls - 1) as f64 - + processing_time) - / total_polls as f64; - - if msgs.is_empty() { - state.processing_stats.empty_polls_count += 1; - } - - let produced_count = msgs.len(); - let total_documents_fetched = state.total_documents_fetched; - drop(state); + let (produced_count, total_documents_fetched) = { + let mut state = self.state.lock().await; + if outcome.messages.is_empty() { + state.processing_stats.empty_polls_count += 1; + } else { + state.processing_stats.successful_polls_count += 1; + state.processing_stats.last_successful_poll = Some(Utc::now()); + } + + let total_polls = state.processing_stats.successful_polls_count + + state.processing_stats.empty_polls_count; + state.processing_stats.avg_batch_processing_time_ms = + (state.processing_stats.avg_batch_processing_time_ms + * (total_polls - 1) as f64 + + processing_time) + / total_polls as f64; + + ( + outcome.messages.len(), + state.total_documents_fetched, + ) + }; if self.verbose { info!( @@ -544,7 +531,7 @@ impl Source for OpenSearchSource { ); } - msgs + outcome.messages } Err(e) => { let mut state = self.state.lock().await; @@ -554,11 +541,14 @@ impl Source for OpenSearchSource { return Err(e); } }; + let persisted_state = { let state = self.state.lock().await; self.serialize_state(&state) }; + sleep(self.polling_interval).await; + Ok(ProducedMessages { schema: Schema::Json, messages, @@ -597,6 +587,9 @@ impl Source for OpenSearchSource { } } +#[cfg(test)] +mod http_tests; + #[cfg(test)] mod tests { use super::*; @@ -621,7 +614,6 @@ mod tests { last_poll_timestamp: None, total_documents_fetched: 500, poll_count: 5, - last_document_id: Some("doc_42".to_string()), search_after: Some(vec![json!("2024-01-01T00:00:00Z"), json!("doc_42")]), error_count: 1, last_error: Some("connection reset".to_string()), @@ -648,8 +640,12 @@ mod tests { let restored = source.state.lock().await; assert_eq!(restored.total_documents_fetched, 500); assert_eq!(restored.poll_count, 5); - assert_eq!(restored.last_document_id, Some("doc_42".to_string())); + assert_eq!( + restored.search_after, + Some(vec![json!("2024-01-01T00:00:00Z"), json!("doc_42")]) + ); assert!(source.state_restore_error.is_none()); + assert!(source.runtime_state_restored); }); } @@ -662,8 +658,9 @@ mod tests { let state = source.state.lock().await; assert_eq!(state.total_documents_fetched, 0); assert_eq!(state.poll_count, 0); - assert_eq!(state.last_document_id, None); + assert_eq!(state.search_after, None); assert!(source.state_restore_error.is_none()); + assert!(!source.runtime_state_restored); }); } @@ -701,6 +698,121 @@ mod tests { assert!(matches!(error, Error::InvalidConfigValue(_))); } + #[test] + fn given_empty_timestamp_field_when_validate_should_fail() { + let mut config = test_config(); + config.timestamp_field = Some(String::new()); + let error = validate_open_config(&config).expect_err("empty timestamp_field"); + assert!(matches!(error, Error::InvalidConfigValue(_))); + } + + #[test] + fn given_unparseable_timestamp_value_should_return_none() { + let value = json!("not-a-timestamp"); + assert!(parse_document_timestamp(&value).is_none()); + } + + #[test] + fn given_source_state_json_when_apply_should_restore_metrics() { + use crate::state_manager::{SOURCE_STATE_VERSION, SourceState}; + + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let mut source = OpenSearchSource::new(1, test_config(), None); + let source_state = SourceState { + id: "opensearch_source_1".to_string(), + last_updated: Utc::now(), + version: SOURCE_STATE_VERSION, + data: json!({ + "total_documents_fetched": 9, + "poll_count": 4, + "search_after": ["2024-02-01T00:00:00Z", "doc-9"], + "error_count": 2, + "last_error": "timeout", + "processing_stats": { + "total_bytes_processed": 100, + "avg_batch_processing_time_ms": 1.5, + "last_successful_poll": null, + "empty_polls_count": 1, + "successful_polls_count": 3 + } + }), + metadata: None, + }; + + source + .source_state_to_internal_state(source_state) + .await + .expect("apply source state"); + + let state = source.state.lock().await; + assert_eq!(state.total_documents_fetched, 9); + assert_eq!(state.poll_count, 4); + assert_eq!( + state.search_after, + Some(vec![json!("2024-02-01T00:00:00Z"), json!("doc-9")]) + ); + assert_eq!(state.error_count, 2); + }); + } + + #[test] + fn given_unsupported_file_state_version_should_fail() { + use crate::state_manager::SourceState; + + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let mut source = OpenSearchSource::new(1, test_config(), None); + let source_state = SourceState { + id: "opensearch_source_1".to_string(), + last_updated: Utc::now(), + version: 99, + data: json!({ "poll_count": 1 }), + metadata: None, + }; + + let error = source + .source_state_to_internal_state(source_state) + .await + .expect_err("unsupported version"); + assert!(matches!(error, Error::Serialization(_))); + }); + } + + #[test] + fn given_invalid_url_when_open_should_return_invalid_config() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let mut config = test_config(); + config.url = "not-a-url".to_string(); + let mut source = OpenSearchSource::new(1, config, None); + let error = source.open().await.expect_err("invalid url"); + assert!(matches!(error, Error::InvalidConfigValue(_))); + }); + } + + #[test] + fn given_internal_state_when_export_should_round_trip_source_state() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let source = OpenSearchSource::new(1, test_config(), None); + { + let mut runtime_state = source.state.lock().await; + *runtime_state = test_state(); + } + let exported = source + .internal_state_to_source_state() + .await + .expect("export state"); + assert_eq!(exported.id, "opensearch_source_1"); + assert_eq!(exported.data["total_documents_fetched"], 500); + assert_eq!( + exported.metadata.as_ref().unwrap()["index"], + "test_documents" + ); + }); + } + #[test] fn given_zero_batch_size_when_validate_should_fail() { let mut config = test_config(); @@ -734,7 +846,6 @@ mod tests { deserialized.total_documents_fetched ); assert_eq!(original.poll_count, deserialized.poll_count); - assert_eq!(original.last_document_id, deserialized.last_document_id); assert_eq!(original.search_after, deserialized.search_after); assert_eq!(original.error_count, deserialized.error_count); } diff --git a/core/connectors/sources/opensearch_source/src/state_manager.rs b/core/connectors/sources/opensearch_source/src/state_manager.rs index 0800db1c53..573bbf744a 100644 --- a/core/connectors/sources/opensearch_source/src/state_manager.rs +++ b/core/connectors/sources/opensearch_source/src/state_manager.rs @@ -22,9 +22,12 @@ use async_trait::async_trait; use iggy_common::{DateTime, Utc}; use iggy_connector_sdk::Error; use serde::{Deserialize, Serialize}; +use std::io::ErrorKind; use std::sync::Arc; use tracing::info; +pub(crate) const SOURCE_STATE_VERSION: u32 = 1; + impl OpenSearchSource { pub(super) async fn save_state(&self) -> Result<(), Error> { if !self @@ -37,12 +40,16 @@ impl OpenSearchSource { return Ok(()); } - let storage = create_state_storage( - self.config - .state - .as_ref() - .expect("state.enabled implies Some(state)"), - )?; + let state_config = self + .config + .state + .as_ref() + .ok_or_else(|| { + Error::InvalidConfigValue( + "plugin_config.state.enabled is true but state config is missing".to_string(), + ) + })?; + let storage = create_state_storage(state_config)?; let source_state = self.internal_state_to_source_state().await?; storage.save_source_state(&source_state).await?; @@ -65,12 +72,16 @@ impl OpenSearchSource { return Ok(()); } - let storage = create_state_storage( - self.config - .state - .as_ref() - .expect("state.enabled implies Some(state)"), - )?; + let state_config = self + .config + .state + .as_ref() + .ok_or_else(|| { + Error::InvalidConfigValue( + "plugin_config.state.enabled is true but state config is missing".to_string(), + ) + })?; + let storage = create_state_storage(state_config)?; let state_id = self.get_state_id(); if let Some(source_state) = storage.load_source_state(&state_id).await? { @@ -99,54 +110,51 @@ impl OpenSearchSource { } } -pub(crate) fn create_state_storage(config: &StateConfig) -> Result, Error> { +pub(crate) fn validate_state_storage_config(config: &StateConfig) -> Result<(), Error> { match config.storage_type.as_deref() { - Some("file") | None => { - let base_path = config - .storage_config - .as_ref() - .and_then(|c| c.get("base_path")) - .and_then(|p| p.as_str()) - .unwrap_or("./connector_states"); - - Ok(Arc::new(FileStateStorage::new(base_path))) - } + Some("file") | None => Ok(()), Some(storage_type) => Err(Error::InvalidConfigValue(format!( "state storage_type {storage_type:?} is not supported; only \"file\" is implemented" ))), } } -/// State management for source connectors +pub(crate) fn create_state_storage(config: &StateConfig) -> Result, Error> { + validate_state_storage_config(config)?; + + let base_path = config + .storage_config + .as_ref() + .and_then(|c| c.get("base_path")) + .and_then(|p| p.as_str()) + .unwrap_or("./connector_states"); + + Ok(Arc::new(FileStateStorage::new(base_path))) +} + +/// Optional file-backed mirror of connector state. Runtime `ConnectorState` msgpack is authoritative. #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SourceState { - /// Unique identifier for this state +pub(crate) struct SourceState { pub id: String, - /// Timestamp when this state was last updated pub last_updated: DateTime, - /// Version of the state format pub version: u32, - /// Generic state data as JSON pub data: serde_json::Value, - /// Optional metadata pub metadata: Option, } -/// State storage backend trait #[async_trait] -pub trait StateStorage: Send + Sync { +pub(crate) trait StateStorage: Send + Sync { async fn save_source_state(&self, state: &SourceState) -> Result<(), Error>; async fn load_source_state(&self, id: &str) -> Result, Error>; } -/// File-based state storage implementation -pub struct FileStateStorage { +pub(crate) struct FileStateStorage { base_path: std::path::PathBuf, } impl FileStateStorage { - pub fn new>(base_path: P) -> Self { + pub(crate) fn new>(base_path: P) -> Self { Self { base_path: base_path.as_ref().to_path_buf(), } @@ -167,12 +175,16 @@ impl StateStorage for FileStateStorage { .map_err(|e| Error::Storage(format!("Failed to create state directory: {e}")))?; let path = self.get_state_path(&state.id); + let tmp_path = path.with_extension("json.tmp"); let json = serde_json::to_string_pretty(state) .map_err(|e| Error::Serialization(format!("Failed to serialize source state: {e}")))?; - fs::write(path, json) + fs::write(&tmp_path, json) + .await + .map_err(|e| Error::Storage(format!("Failed to write state temp file: {e}")))?; + fs::rename(&tmp_path, &path) .await - .map_err(|e| Error::Storage(format!("Failed to write state file: {e}")))?; + .map_err(|e| Error::Storage(format!("Failed to rename state file: {e}")))?; Ok(()) } @@ -181,13 +193,13 @@ impl StateStorage for FileStateStorage { use tokio::fs; let path = self.get_state_path(id); - if !path.exists() { - return Ok(None); - } - - let content = fs::read_to_string(path) - .await - .map_err(|e| Error::Storage(format!("Failed to read state file: {e}")))?; + let content = match fs::read_to_string(&path).await { + Ok(content) => content, + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(Error::Storage(format!("Failed to read state file: {error}"))); + } + }; let state: SourceState = serde_json::from_str(&content).map_err(|e| { Error::Serialization(format!("Failed to deserialize source state: {e}")) @@ -196,3 +208,106 @@ impl StateStorage for FileStateStorage { Ok(Some(state)) } } + +#[cfg(test)] +mod tests { + use super::*; + use iggy_common::Utc; + use serde_json::json; + use tempfile::TempDir; + + fn file_state_config(base_path: &str) -> StateConfig { + StateConfig { + enabled: true, + storage_type: Some("file".to_string()), + storage_config: Some(json!({ "base_path": base_path })), + state_id: Some("opensearch_unit_state".to_string()), + } + } + + #[test] + fn given_unknown_storage_type_should_fail() { + let config = StateConfig { + enabled: true, + storage_type: Some("s3".to_string()), + storage_config: None, + state_id: None, + }; + let error = validate_state_storage_config(&config); + assert!(matches!(error, Err(Error::InvalidConfigValue(_)))); + } + + #[test] + fn given_file_storage_should_save_and_load_source_state() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let temp_dir = TempDir::new().expect("tempdir"); + let config = file_state_config(&temp_dir.path().to_string_lossy()); + let storage = create_state_storage(&config).expect("file storage"); + + let source_state = SourceState { + id: "opensearch_unit_state".to_string(), + last_updated: Utc::now(), + version: SOURCE_STATE_VERSION, + data: json!({ + "total_documents_fetched": 7, + "poll_count": 2, + "search_after": ["2024-01-01T00:00:00Z", "doc-7"] + }), + metadata: None, + }; + + storage + .save_source_state(&source_state) + .await + .expect("save state"); + let loaded = storage + .load_source_state("opensearch_unit_state") + .await + .expect("load state") + .expect("state file should exist"); + assert_eq!(loaded.data["total_documents_fetched"], 7); + assert_eq!(loaded.data["poll_count"], 2); + }); + } + + #[test] + fn given_missing_state_file_when_load_should_return_none() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let temp_dir = TempDir::new().expect("tempdir"); + let config = file_state_config(&temp_dir.path().to_string_lossy()); + let storage = create_state_storage(&config).expect("file storage"); + let loaded = storage + .load_source_state("missing_state_id") + .await + .expect("load should not error"); + assert!(loaded.is_none()); + }); + } + + #[test] + fn given_default_storage_type_should_use_file_backend() { + let config = StateConfig { + enabled: true, + storage_type: None, + storage_config: None, + state_id: None, + }; + let storage = create_state_storage(&config).expect("default file storage"); + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let source_state = SourceState { + id: "opensearch_default".to_string(), + last_updated: Utc::now(), + version: SOURCE_STATE_VERSION, + data: json!({ "poll_count": 1 }), + metadata: None, + }; + storage + .save_source_state(&source_state) + .await + .expect("save"); + }); + } +} diff --git a/core/integration/tests/connectors/fixtures/mod.rs b/core/integration/tests/connectors/fixtures/mod.rs index 4fac6db286..d8f18a6dee 100644 --- a/core/integration/tests/connectors/fixtures/mod.rs +++ b/core/integration/tests/connectors/fixtures/mod.rs @@ -69,7 +69,10 @@ pub use mongodb::{ MongoDbOps, MongoDbSinkAutoCreateFixture, MongoDbSinkBatchFixture, MongoDbSinkFailpointFixture, MongoDbSinkFixture, MongoDbSinkJsonFixture, MongoDbSinkWriteConcernFixture, }; -pub use opensearch::{OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture}; +pub use opensearch::{ + OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture, + OpenSearchSourceTypedFieldsFixture, +}; pub use postgres::{ PostgresOps, PostgresSinkByteaFixture, PostgresSinkFixture, PostgresSinkJsonFixture, PostgresSourceByteaFixture, PostgresSourceDeleteFixture, PostgresSourceJsonFixture, diff --git a/core/integration/tests/connectors/fixtures/opensearch/container.rs b/core/integration/tests/connectors/fixtures/opensearch/container.rs index 245acab021..b8aa480889 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/container.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/container.rs @@ -17,6 +17,7 @@ * under the License. */ +use crate::connectors::fixtures; use integration::harness::TestBinaryError; use reqwest_middleware::ClientWithMiddleware as HttpClient; use reqwest_retry::RetryTransientMiddleware; @@ -25,20 +26,13 @@ use serde::Deserialize; use testcontainers_modules::testcontainers::core::wait::HttpWaitStrategy; use testcontainers_modules::testcontainers::core::{IntoContainerPort, WaitFor}; use testcontainers_modules::testcontainers::runners::AsyncRunner; -use testcontainers_modules::testcontainers::{ - ContainerAsync, GenericImage, ImageExt, ReuseDirective, -}; +use testcontainers_modules::testcontainers::{ContainerAsync, GenericImage, ImageExt}; use tracing::info; const OPENSEARCH_IMAGE: &str = "docker.io/opensearchproject/opensearch"; const OPENSEARCH_TAG: &str = "2.19.1"; const OPENSEARCH_PORT: u16 = 9200; const OPENSEARCH_HEALTH_ENDPOINT: &str = "/_cluster/health"; -// Fixed name + ReuseDirective::Always shares one container across nextest's -// per-test processes: the first test creates it, every later test attaches by -// name. Per-test isolation comes from a unique index per fixture, not a fresh -// container. -const OPENSEARCH_CONTAINER_NAME: &str = "iggy-test-opensearch"; pub const DEFAULT_TEST_STREAM: &str = "test_stream"; pub const DEFAULT_TEST_TOPIC: &str = "test_topic"; @@ -83,8 +77,6 @@ pub struct OpenSearchHit { } pub struct OpenSearchContainer { - // Held so testcontainers' Drop runs on test exit; ReuseDirective::Always - // makes that Drop leave the container running for the next test to attach. #[allow(dead_code)] container: ContainerAsync, pub base_url: String, @@ -102,10 +94,11 @@ impl OpenSearchContainer { .with_startup_timeout(std::time::Duration::from_secs(120)) .with_env_var("discovery.type", "single-node") .with_env_var("plugins.security.disabled", "true") + .with_env_var("DISABLE_INSTALL_DEMO_CONFIG", "true") + .with_env_var("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "iggy-test-password1!") .with_env_var("OPENSEARCH_JAVA_OPTS", "-Xms512m -Xmx512m") .with_mapped_port(0, OPENSEARCH_PORT.tcp()) - .with_container_name(OPENSEARCH_CONTAINER_NAME) - .with_reuse(ReuseDirective::Always) + .with_container_name(fixtures::unique_container_name("opensearch")) .start() .await .map_err(|e| TestBinaryError::FixtureSetup { @@ -196,6 +189,57 @@ pub trait OpenSearchOps: Sync { } } + fn create_typed_fields_index( + &self, + index_name: &str, + ) -> impl std::future::Future> + Send { + async move { + let url = format!("{}/{}", self.container().base_url, index_name); + let mapping = serde_json::json!({ + "mappings": { + "properties": { + "id": { "type": "integer" }, + "title": { "type": "text" }, + "status": { "type": "keyword" }, + "count": { "type": "long" }, + "score": { "type": "float" }, + "ratio": { "type": "double" }, + "active": { "type": "boolean" }, + "timestamp": { "type": "date" }, + "client_ip": { "type": "ip" }, + "location": { "type": "geo_point" }, + "tags": { "type": "keyword" }, + "optional_note": { "type": "keyword" } + } + } + }); + + let response = self + .http_client() + .put(&url) + .header("Content-Type", "application/json") + .json(&mapping) + .send() + .await + .map_err(|e| TestBinaryError::FixtureSetup { + fixture_type: "OpenSearchOps".to_string(), + message: format!("Failed to create typed index: {e}"), + })?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(TestBinaryError::FixtureSetup { + fixture_type: "OpenSearchOps".to_string(), + message: format!("Failed to create typed index: status={status}, body={body}"), + }); + } + + info!("Created typed OpenSearch index: {index_name}"); + Ok(()) + } + } + fn index_document( &self, index_name: &str, diff --git a/core/integration/tests/connectors/fixtures/opensearch/mod.rs b/core/integration/tests/connectors/fixtures/opensearch/mod.rs index 963cbfb305..a303d84460 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/mod.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/mod.rs @@ -20,4 +20,7 @@ pub mod container; pub mod source; -pub use source::{OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture}; +pub use source::{ + OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture, + OpenSearchSourceTypedFieldsFixture, +}; diff --git a/core/integration/tests/connectors/fixtures/opensearch/source.rs b/core/integration/tests/connectors/fixtures/opensearch/source.rs index 8df76c3ef9..66fe0cc0a6 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/source.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/source.rs @@ -36,8 +36,7 @@ const TEST_INDEX_PREFIX: &str = "test_documents"; pub struct OpenSearchSourceFixture { container: OpenSearchContainer, http_client: HttpClient, - // Unique per fixture so tests sharing one container never collide on the - // same index. The connector reads from here via ENV_SOURCE_INDEX. + // Unique per fixture so parallel tests never collide on the same index. index: String, } @@ -97,6 +96,27 @@ impl OpenSearchSourceFixture { pub async fn refresh_index(&self) -> Result<(), TestBinaryError> { OpenSearchOps::refresh_index(self, &self.index).await } + + pub async fn insert_typed_sample_document(&self) -> Result<(), TestBinaryError> { + let timestamp = IggyTimestamp::now().to_rfc3339_string(); + let document = serde_json::json!({ + "id": 1, + "title": "OpenSearch typed field coverage", + "status": "active", + "count": 9_223_372_036_854_775_807_i64, + "score": 98.6_f32, + "ratio": 0.125_f64, + "active": true, + "timestamp": timestamp, + "client_ip": "192.168.1.42", + "location": { "lat": 40.12, "lon": -71.34 }, + "tags": ["integration", "opensearch"], + "optional_note": null + }); + self.index_document(&self.index, "typed-1", &document) + .await?; + self.refresh_index().await + } } #[async_trait] @@ -142,6 +162,41 @@ impl TestFixture for OpenSearchSourceFixture { } } +/// OpenSearch source fixture with typed-field index mapping. +pub struct OpenSearchSourceTypedFieldsFixture { + inner: OpenSearchSourceFixture, +} + +impl std::ops::Deref for OpenSearchSourceTypedFieldsFixture { + type Target = OpenSearchSourceFixture; + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl OpenSearchOps for OpenSearchSourceTypedFieldsFixture { + fn container(&self) -> &OpenSearchContainer { + &self.inner.container + } + + fn http_client(&self) -> &HttpClient { + &self.inner.http_client + } +} + +#[async_trait] +impl TestFixture for OpenSearchSourceTypedFieldsFixture { + async fn setup() -> Result { + let inner = OpenSearchSourceFixture::setup().await?; + inner.create_typed_fields_index(&inner.index).await?; + Ok(Self { inner }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + self.inner.connectors_runtime_envs() + } +} + /// OpenSearch source fixture with pre-created index. pub struct OpenSearchSourcePreCreatedFixture { inner: OpenSearchSourceFixture, diff --git a/core/integration/tests/connectors/opensearch/docker-compose.yml b/core/integration/tests/connectors/opensearch/docker-compose.yml index fa01478b94..e765a8b006 100644 --- a/core/integration/tests/connectors/opensearch/docker-compose.yml +++ b/core/integration/tests/connectors/opensearch/docker-compose.yml @@ -23,18 +23,20 @@ # Used as a fallback when running tests without a Docker daemon that supports # testcontainers auto-launch (e.g. some CI environments). Start manually with: # docker compose -f core/integration/tests/connectors/opensearch/docker-compose.yml up -d -# then run: +# then run (set IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_URL=http://localhost:9200): # cargo test -p integration -- connectors::opensearch services: opensearch: image: opensearchproject/opensearch:2.19.1 - container_name: iggy-test-opensearch + container_name: iggy-opensearch-manual ports: - "9200:9200" environment: discovery.type: single-node plugins.security.disabled: "true" + DISABLE_INSTALL_DEMO_CONFIG: "true" + OPENSEARCH_INITIAL_ADMIN_PASSWORD: "iggy-test-password1!" OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m" healthcheck: test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"] diff --git a/core/integration/tests/connectors/opensearch/mod.rs b/core/integration/tests/connectors/opensearch/mod.rs index 742b93ee3e..abf01cea52 100644 --- a/core/integration/tests/connectors/opensearch/mod.rs +++ b/core/integration/tests/connectors/opensearch/mod.rs @@ -18,6 +18,7 @@ */ mod opensearch_source; +mod opensearch_source_types; const TEST_MESSAGE_COUNT: usize = 3; const POLL_ATTEMPTS: usize = 100; diff --git a/core/integration/tests/connectors/opensearch/opensearch_source.rs b/core/integration/tests/connectors/opensearch/opensearch_source.rs index c52396628a..aba5bdfcb9 100644 --- a/core/integration/tests/connectors/opensearch/opensearch_source.rs +++ b/core/integration/tests/connectors/opensearch/opensearch_source.rs @@ -48,6 +48,44 @@ fn assert_contains_document_ids(messages: &[serde_json::Value], expected_ids: &[ } } +async fn poll_all_messages_from_offset_zero( + client: &impl MessageClient, + consumer_id: &Identifier, + min_messages: usize, +) -> Vec { + let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); + let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); + let mut received = Vec::new(); + + for _ in 0..POLL_ATTEMPTS { + if let Ok(polled) = client + .poll_messages( + &stream_id, + &topic_id, + None, + &Consumer::new(consumer_id.clone()), + &PollingStrategy::offset(0), + 100, + false, + ) + .await + { + received.clear(); + for msg in polled.messages { + if let Ok(json) = serde_json::from_slice(&msg.payload) { + received.push(json); + } + } + if received.len() >= min_messages { + break; + } + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + + received +} + #[iggy_harness( server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), seed = seeds::connector_stream @@ -281,41 +319,33 @@ async fn given_runtime_state_when_connector_restarts_should_resume_after_cursor( .expect("Failed to restart connectors"); sleep(Duration::from_millis(100)).await; - let mut received_after: Vec = Vec::new(); - for _ in 0..POLL_ATTEMPTS { - if let Ok(polled) = client - .poll_messages( - &stream_id, - &topic_id, - None, - &Consumer::new(consumer_id.clone()), - &PollingStrategy::next(), - 10, - true, - ) - .await - { - for msg in polled.messages { - if let Ok(json) = serde_json::from_slice(&msg.payload) { - received_after.push(json); - } - } - if received_after.len() >= TEST_MESSAGE_COUNT { - break; - } - } - sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; - } + let audit_consumer: Identifier = "state_audit_consumer".try_into().unwrap(); + let all_messages = + poll_all_messages_from_offset_zero(&client, &audit_consumer, TEST_MESSAGE_COUNT * 2).await; - assert_eq!(received_after.len(), TEST_MESSAGE_COUNT); + let batch1_ids: HashSet = (1..=TEST_MESSAGE_COUNT as i64).collect(); + let batch1_occurrences = all_messages + .iter() + .filter_map(|record| record.get("id").and_then(|value| value.as_i64())) + .filter(|id| batch1_ids.contains(id)) + .count(); + assert_eq!( + batch1_occurrences, TEST_MESSAGE_COUNT, + "batch 1 IDs must appear exactly once on the stream; duplicates mean cursor reset" + ); - for record in &received_after { - let id = record.get("id").and_then(|v| v.as_i64()).unwrap_or(0); - assert!( - id > TEST_MESSAGE_COUNT as i64, - "After restart, got ID {id} from first batch" - ); - } + let batch2_ids: HashSet = + ((TEST_MESSAGE_COUNT + 1) as i64..=(TEST_MESSAGE_COUNT * 2) as i64).collect(); + let batch2_seen: HashSet = all_messages + .iter() + .filter_map(|record| record.get("id").and_then(|value| value.as_i64())) + .filter(|id| batch2_ids.contains(id)) + .collect(); + assert_eq!( + batch2_seen.len(), + TEST_MESSAGE_COUNT, + "batch 2 IDs must be present after restart, got {batch2_seen:?}" + ); } async fn fetch_sources(http_client: &Client, api_address: &str) -> Vec { @@ -362,8 +392,8 @@ async fn given_missing_index_when_connector_opens_should_report_error( .as_ref() .expect("Source with missing index should expose a last_error"); assert!( - last_error.message.contains("does not exist"), - "last_error should mention the missing index, got: {}", + last_error.message.contains("Plugin initialization failed"), + "missing index should fail during plugin open, got: {}", last_error.message ); } diff --git a/core/integration/tests/connectors/opensearch/opensearch_source_types.rs b/core/integration/tests/connectors/opensearch/opensearch_source_types.rs new file mode 100644 index 0000000000..0ff37fa992 --- /dev/null +++ b/core/integration/tests/connectors/opensearch/opensearch_source_types.rs @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use super::{POLL_ATTEMPTS, POLL_INTERVAL_MS, TEST_MESSAGE_COUNT}; +use crate::connectors::fixtures::{ + OpenSearchSourcePreCreatedFixture, OpenSearchSourceTypedFieldsFixture, +}; +use iggy_common::MessageClient; +use iggy_common::{Consumer, Identifier, PollingStrategy}; +use integration::harness::seeds; +use integration::iggy_harness; +use serde_json::Value; +use std::collections::HashSet; +use std::time::Duration; +use tokio::time::sleep; + +async fn poll_json_messages( + client: &impl MessageClient, + consumer_id: &Identifier, + limit: u32, +) -> Vec { + let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); + let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); + let mut received = Vec::new(); + + for _ in 0..POLL_ATTEMPTS { + if let Ok(polled) = client + .poll_messages( + &stream_id, + &topic_id, + None, + &Consumer::new(consumer_id.clone()), + &PollingStrategy::next(), + limit, + true, + ) + .await + { + for message in polled.messages { + if let Ok(json) = serde_json::from_slice::(&message.payload) { + received.push(json); + } + } + if !received.is_empty() { + break; + } + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + + received +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), + seed = seeds::connector_stream +)] +async fn opensearch_source_message_payload_structure( + harness: &TestHarness, + fixture: OpenSearchSourcePreCreatedFixture, +) { + fixture + .insert_document(1, "structure_doc", 42) + .await + .expect("insert document"); + fixture.refresh_index().await.expect("refresh index"); + + let client = harness.root_client().await.unwrap(); + let consumer_id: Identifier = "payload_structure_consumer".try_into().unwrap(); + let messages = poll_json_messages(&client, &consumer_id, 10).await; + + assert_eq!( + messages.len(), + 1, + "expected one message, got {}", + messages.len() + ); + let record = &messages[0]; + assert_eq!(record["id"], 1); + assert_eq!(record["name"], "structure_doc"); + assert_eq!(record["value"], 42); + assert!(record.get("timestamp").is_some(), "missing timestamp field"); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), + seed = seeds::connector_stream +)] +async fn opensearch_source_typed_fields_should_round_trip_in_payload( + harness: &TestHarness, + fixture: OpenSearchSourceTypedFieldsFixture, +) { + fixture + .insert_typed_sample_document() + .await + .expect("insert typed document"); + + let client = harness.root_client().await.unwrap(); + let consumer_id: Identifier = "typed_fields_consumer".try_into().unwrap(); + let messages = poll_json_messages(&client, &consumer_id, 10).await; + + assert_eq!(messages.len(), 1, "expected one typed message"); + let record = &messages[0]; + assert_eq!(record["title"], "OpenSearch typed field coverage"); + assert_eq!(record["status"], "active"); + assert_eq!(record["count"].as_i64(), Some(9_223_372_036_854_775_807)); + assert!((record["score"].as_f64().unwrap() - 98.6).abs() < 0.01); + assert!((record["ratio"].as_f64().unwrap() - 0.125).abs() < f64::EPSILON); + assert_eq!(record["active"], true); + assert_eq!(record["client_ip"], "192.168.1.42"); + assert_eq!(record["location"]["lat"], 40.12); + assert_eq!(record["location"]["lon"], -71.34); + assert!(record["tags"].is_array()); + assert!(record["optional_note"].is_null()); + assert!(record.get("timestamp").is_some()); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), + seed = seeds::connector_stream +)] +async fn opensearch_source_search_after_when_second_batch_inserted_should_not_duplicate( + harness: &TestHarness, + fixture: OpenSearchSourcePreCreatedFixture, +) { + fixture + .insert_documents(TEST_MESSAGE_COUNT) + .await + .expect("insert first batch"); + + let client = harness.root_client().await.unwrap(); + let consumer_id: Identifier = "cursor_consumer".try_into().unwrap(); + + let first_batch = poll_json_messages(&client, &consumer_id, 10).await; + assert_eq!(first_batch.len(), TEST_MESSAGE_COUNT); + let first_ids: HashSet = first_batch + .iter() + .filter_map(|record| record.get("id").and_then(Value::as_i64)) + .collect(); + + let second_batch_start_id = (TEST_MESSAGE_COUNT + 1) as i32; + for offset in 0..TEST_MESSAGE_COUNT { + fixture + .insert_document( + second_batch_start_id + offset as i32, + &format!("batch_two_{offset}"), + (100 + offset) as i32, + ) + .await + .expect("insert second batch document"); + } + fixture.refresh_index().await.expect("refresh index"); + + let mut second_batch = Vec::new(); + for _ in 0..POLL_ATTEMPTS { + let polled = poll_json_messages(&client, &consumer_id, 10).await; + for record in polled { + let id = record.get("id").and_then(Value::as_i64).unwrap_or(0); + if !first_ids.contains(&id) { + second_batch.push(record); + } + } + if second_batch.len() >= TEST_MESSAGE_COUNT { + break; + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + + assert_eq!( + second_batch.len(), + TEST_MESSAGE_COUNT, + "expected second batch only" + ); + for record in &second_batch { + let id = record.get("id").and_then(Value::as_i64).unwrap_or(0); + assert!( + id > TEST_MESSAGE_COUNT as i64, + "duplicate or first-batch id in second batch: {id}" + ); + } +} From bab1969f47350245186856e6454e86696e344612 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Thu, 18 Jun 2026 09:29:32 -0400 Subject: [PATCH 06/19] Code review refactors OpenSearch source state & polling logic Major refactor of the OpenSearch source: enforce and skip hits missing sort tuples (log a warning), and move search/poll bookkeeping into a new finalize_poll flow that returns produced messages and a serialized state. Track and persist additional processing stats (avg_batch_processing_time_ms, empty/successful poll counts, last_successful_poll, total_bytes_processed) and compute running average across restarts. Improve state storage: simplify enabled-checks, use atomic temp file write with OpenOptions + sync_data, better error messages, and use ConnectorState.deserialize helper for restores. Add a small-batch test fixture and an integration test for pagination (fetching across search_after pages). Misc: delete default connector state JSON, update dependencies.md note for humantime, add unit test for skipping hits without sort, and rename/clean up several test functions for clarity. --- .../connector_states/opensearch_default.json | 9 -- .../sources/opensearch_source/dependencies.md | 2 +- .../opensearch_source/src/http_tests.rs | 28 +++- .../sources/opensearch_source/src/lib.rs | 141 +++++++++--------- .../opensearch_source/src/state_manager.rs | 68 ++++----- .../tests/connectors/fixtures/mod.rs | 2 +- .../connectors/fixtures/opensearch/mod.rs | 2 +- .../connectors/fixtures/opensearch/source.rs | 38 +++++ .../opensearch/opensearch_source.rs | 58 +++++++ .../opensearch/opensearch_source_types.rs | 6 +- 10 files changed, 225 insertions(+), 129 deletions(-) delete mode 100644 core/connectors/sources/opensearch_source/connector_states/opensearch_default.json diff --git a/core/connectors/sources/opensearch_source/connector_states/opensearch_default.json b/core/connectors/sources/opensearch_source/connector_states/opensearch_default.json deleted file mode 100644 index 205c85ebc5..0000000000 --- a/core/connectors/sources/opensearch_source/connector_states/opensearch_default.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "id": "opensearch_default", - "last_updated": "2026-06-17T23:48:32.264826Z", - "version": 1, - "data": { - "poll_count": 1 - }, - "metadata": null -} \ No newline at end of file diff --git a/core/connectors/sources/opensearch_source/dependencies.md b/core/connectors/sources/opensearch_source/dependencies.md index fc61ce8dd2..e020a313c9 100644 --- a/core/connectors/sources/opensearch_source/dependencies.md +++ b/core/connectors/sources/opensearch_source/dependencies.md @@ -33,7 +33,7 @@ to `cargo tree -p iggy_connector_opensearch_source` for the full graph. | --- | --- | --- | --- | | `async-trait` | `^0.1.89` | MIT / Apache-2.0 | Proc-macro that enables `async fn` in trait definitions; required by the `Source` trait impl in `lib.rs`. | | `dashmap` | `^6.1.0` | MIT | Concurrent hash map; injected into this crate's namespace by the `source_connector!` macro expansion in the SDK. Not used directly in source files. | -| `humantime` | `^2.3.0` | MIT / Apache-2.0 | Parses human-readable duration strings in optional file-state `auto_save_interval` config (reserved for future use). | +| `humantime` | `^2.3.0` | MIT / Apache-2.0 | Workspace dependency; duration parsing uses `iggy_connector_sdk::retry::parse_duration`. | | `iggy_common` | workspace | Apache-2.0 | Shared Iggy types: `DateTime`, `Utc`, and `serde_secret` for optional basic-auth password serialisation. | | `iggy_connector_sdk` | workspace | Apache-2.0 | Core connector abstractions: `Source` trait, `ProducedMessage`, `ProducedMessages`, `ConnectorState`, `Schema`, `Error`, `parse_duration`, and the `source_connector!` registration macro. | | `once_cell` | `^1.21.4` | MIT / Apache-2.0 | `Lazy` global; injected by the `source_connector!` macro expansion in the SDK. Not used directly in source files. | diff --git a/core/connectors/sources/opensearch_source/src/http_tests.rs b/core/connectors/sources/opensearch_source/src/http_tests.rs index 3370837916..7d8b575e47 100644 --- a/core/connectors/sources/opensearch_source/src/http_tests.rs +++ b/core/connectors/sources/opensearch_source/src/http_tests.rs @@ -294,6 +294,29 @@ async fn given_basic_auth_when_search_should_send_authorization_header() { assert!(auth.starts_with("Basic ")); } +#[tokio::test] +async fn given_hit_without_sort_when_poll_should_skip_document() { + let hit = json!({ + "_id": "doc-1", + "_source": { "id": 1, "timestamp": "2024-01-01T00:00:00Z" } + }); + let body = search_response(vec![hit]).to_string(); + let app = mock_router(StatusCode::OK, move |_| { + let body = body.clone(); + async move { (StatusCode::OK, body) } + }); + let base = start_server(app).await; + let mut source = OpenSearchSource::new(1, base_config(&base), None); + source.open().await.unwrap(); + + let produced = source.poll().await.expect("poll should succeed"); + assert!(produced.messages.is_empty()); + + let (_, polls, _, empty_polls) = source.test_metrics().await; + assert_eq!(polls, 1); + assert_eq!(empty_polls, 1); +} + #[tokio::test] async fn given_hit_without_source_when_search_should_skip_document() { let hit = json!({ @@ -411,7 +434,10 @@ async fn given_runtime_state_when_open_should_not_load_stale_file_state() { restarted.open().await.unwrap(); let (_, polls, _, _) = restarted.test_metrics().await; - assert_eq!(polls, 1, "runtime ConnectorState must not be overwritten by stale file"); + assert_eq!( + polls, 1, + "runtime ConnectorState must not be overwritten by stale file" + ); } #[tokio::test] diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs index f71bfaedac..2a4fe7bf37 100644 --- a/core/connectors/sources/opensearch_source/src/lib.rs +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -35,9 +35,7 @@ use tokio::{sync::Mutex, time::sleep}; use tracing::{debug, error, info, warn}; mod state_manager; -use crate::state_manager::{ - SOURCE_STATE_VERSION, SourceState, validate_state_storage_config, -}; +use crate::state_manager::{SOURCE_STATE_VERSION, SourceState, validate_state_storage_config}; source_connector!(OpenSearchSource); @@ -68,6 +66,8 @@ struct State { struct ProcessingStats { #[serde(default)] total_bytes_processed: u64, + /// Running cumulative average over the connector's lifetime, persisted and accumulated + /// across restarts. Reflects long-term throughput baseline, not session-only average. #[serde(default)] avg_batch_processing_time_ms: f64, #[serde(default)] @@ -270,32 +270,39 @@ impl OpenSearchSource { ))); } - let response_body: Value = response + let mut response_body: Value = response .json() .await .map_err(|e| Error::Storage(format!("Failed to parse search response: {e}")))?; - let hits = response_body - .get("hits") - .and_then(|h| h.get("hits")) - .and_then(|h| h.as_array()) - .cloned() + let hits: Vec = response_body + .get_mut("hits") + .and_then(|h| h.get_mut("hits")) + .and_then(|arr| arr.as_array_mut()) + .map(std::mem::take) .unwrap_or_default(); let mut messages = Vec::with_capacity(hits.len()); let mut batch_bytes = 0u64; - let mut last_search_after = None; + let mut last_sort: Option<&Vec> = None; let mut last_poll_timestamp = None; for hit in &hits { - if let Some(sort) = hit.get("sort").and_then(|s| s.as_array()) { - last_search_after = Some(sort.clone()); - } + let Some(sort) = hit.get("sort").and_then(|s| s.as_array()) else { + warn!( + connector_id = self.id, + hit_id = hit.get("_id").and_then(|value| value.as_str()), + "Skipping OpenSearch hit without sort tuple; document will not be published" + ); + continue; + }; let Some(source) = hit.get("_source") else { continue; }; + last_sort = Some(sort); + if let Some(timestamp_value) = source.get(timestamp_field) && let Some(timestamp_utc) = parse_document_timestamp(timestamp_value) { @@ -318,21 +325,61 @@ impl OpenSearchSource { Ok(SearchOutcome { messages, - search_after: last_search_after, + search_after: last_sort.map(ToOwned::to_owned), last_poll_timestamp, batch_bytes, }) } - async fn apply_search_outcome(&self, outcome: &SearchOutcome) { + async fn finalize_poll( + &self, + outcome: SearchOutcome, + processing_time_ms: f64, + ) -> (Vec, Option) { let mut state = self.state.lock().await; state.total_documents_fetched += outcome.messages.len(); state.poll_count += 1; - state.search_after = outcome.search_after.clone(); + state.search_after = outcome.search_after; if let Some(timestamp) = outcome.last_poll_timestamp { state.last_poll_timestamp = Some(timestamp); } state.processing_stats.total_bytes_processed += outcome.batch_bytes; + + if outcome.messages.is_empty() { + state.processing_stats.empty_polls_count += 1; + } else { + state.processing_stats.successful_polls_count += 1; + state.processing_stats.last_successful_poll = Some(Utc::now()); + } + + let total_polls = state.processing_stats.successful_polls_count + + state.processing_stats.empty_polls_count; + state.processing_stats.avg_batch_processing_time_ms = + (state.processing_stats.avg_batch_processing_time_ms * (total_polls - 1) as f64 + + processing_time_ms) + / total_polls as f64; + + let produced_count = outcome.messages.len(); + let total_documents_fetched = state.total_documents_fetched; + let messages = outcome.messages; + let persisted_state = self.serialize_state(&state); + drop(state); + + if self.verbose { + info!( + "OpenSearch source connector ID: {} produced {produced_count} messages. \ + Total fetched: {total_documents_fetched}", + self.id + ); + } else { + debug!( + "OpenSearch source connector ID: {} produced {produced_count} messages. \ + Total fetched: {total_documents_fetched}", + self.id + ); + } + + (messages, persisted_state) } #[cfg(test)] @@ -352,16 +399,12 @@ impl OpenSearchSource { } } -fn restore_state( - id: u32, - state: Option, -) -> (State, Option, bool) { +fn restore_state(id: u32, state: Option) -> (State, Option, bool) { let Some(connector_state) = state else { return (State::default(), None, false); }; - let bytes = connector_state.0; - match ConnectorState(bytes.clone()).deserialize::(CONNECTOR_NAME, id) { + match connector_state.deserialize::(CONNECTOR_NAME, id) { Some(restored) => { info!( "Restored state for {CONNECTOR_NAME} connector with ID: {id}. \ @@ -489,49 +532,10 @@ impl Source for OpenSearchSource { .as_ref() .ok_or_else(|| Error::Storage("OpenSearch client not initialized".to_string()))?; - let messages = match self.search_documents(client).await { + let (messages, persisted_state) = match self.search_documents(client).await { Ok(outcome) => { - self.apply_search_outcome(&outcome).await; - let processing_time = start_time.elapsed().as_millis() as f64; - let (produced_count, total_documents_fetched) = { - let mut state = self.state.lock().await; - if outcome.messages.is_empty() { - state.processing_stats.empty_polls_count += 1; - } else { - state.processing_stats.successful_polls_count += 1; - state.processing_stats.last_successful_poll = Some(Utc::now()); - } - - let total_polls = state.processing_stats.successful_polls_count - + state.processing_stats.empty_polls_count; - state.processing_stats.avg_batch_processing_time_ms = - (state.processing_stats.avg_batch_processing_time_ms - * (total_polls - 1) as f64 - + processing_time) - / total_polls as f64; - - ( - outcome.messages.len(), - state.total_documents_fetched, - ) - }; - - if self.verbose { - info!( - "OpenSearch source connector ID: {} produced {produced_count} messages. \ - Total fetched: {total_documents_fetched}", - self.id - ); - } else { - debug!( - "OpenSearch source connector ID: {} produced {produced_count} messages. \ - Total fetched: {total_documents_fetched}", - self.id - ); - } - - outcome.messages + self.finalize_poll(outcome, processing_time).await } Err(e) => { let mut state = self.state.lock().await; @@ -542,11 +546,6 @@ impl Source for OpenSearchSource { } }; - let persisted_state = { - let state = self.state.lock().await; - self.serialize_state(&state) - }; - sleep(self.polling_interval).await; Ok(ProducedMessages { @@ -570,12 +569,8 @@ impl Source for OpenSearchSource { .as_ref() .map(|s| s.enabled) .unwrap_or(false) - && let Err(e) = self.save_state().await { - warn!( - "Failed to save final state for OpenSearch source connector with ID: {}: {}", - self.id, e - ); + self.save_state().await?; } self.client = None; diff --git a/core/connectors/sources/opensearch_source/src/state_manager.rs b/core/connectors/sources/opensearch_source/src/state_manager.rs index 573bbf744a..72b98a0d5b 100644 --- a/core/connectors/sources/opensearch_source/src/state_manager.rs +++ b/core/connectors/sources/opensearch_source/src/state_manager.rs @@ -30,25 +30,9 @@ pub(crate) const SOURCE_STATE_VERSION: u32 = 1; impl OpenSearchSource { pub(super) async fn save_state(&self) -> Result<(), Error> { - if !self - .config - .state - .as_ref() - .map(|s| s.enabled) - .unwrap_or(false) - { + let Some(state_config) = self.config.state.as_ref().filter(|s| s.enabled) else { return Ok(()); - } - - let state_config = self - .config - .state - .as_ref() - .ok_or_else(|| { - Error::InvalidConfigValue( - "plugin_config.state.enabled is true but state config is missing".to_string(), - ) - })?; + }; let storage = create_state_storage(state_config)?; let source_state = self.internal_state_to_source_state().await?; @@ -62,25 +46,9 @@ impl OpenSearchSource { } pub(super) async fn load_state(&mut self) -> Result<(), Error> { - if !self - .config - .state - .as_ref() - .map(|s| s.enabled) - .unwrap_or(false) - { + let Some(state_config) = self.config.state.as_ref().filter(|s| s.enabled) else { return Ok(()); - } - - let state_config = self - .config - .state - .as_ref() - .ok_or_else(|| { - Error::InvalidConfigValue( - "plugin_config.state.enabled is true but state config is missing".to_string(), - ) - })?; + }; let storage = create_state_storage(state_config)?; let state_id = self.get_state_id(); @@ -168,7 +136,8 @@ impl FileStateStorage { #[async_trait] impl StateStorage for FileStateStorage { async fn save_source_state(&self, state: &SourceState) -> Result<(), Error> { - use tokio::fs; + use tokio::fs::{self, OpenOptions}; + use tokio::io::AsyncWriteExt; fs::create_dir_all(&self.base_path) .await @@ -179,9 +148,23 @@ impl StateStorage for FileStateStorage { let json = serde_json::to_string_pretty(state) .map_err(|e| Error::Serialization(format!("Failed to serialize source state: {e}")))?; - fs::write(&tmp_path, json) + let mut tmp_file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&tmp_path) + .await + .map_err(|e| Error::Storage(format!("Failed to open state temp file: {e}")))?; + tmp_file + .write_all(json.as_bytes()) .await .map_err(|e| Error::Storage(format!("Failed to write state temp file: {e}")))?; + tmp_file + .sync_data() + .await + .map_err(|e| Error::Storage(format!("Failed to sync state temp file: {e}")))?; + drop(tmp_file); + fs::rename(&tmp_path, &path) .await .map_err(|e| Error::Storage(format!("Failed to rename state file: {e}")))?; @@ -197,7 +180,9 @@ impl StateStorage for FileStateStorage { Ok(content) => content, Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None), Err(error) => { - return Err(Error::Storage(format!("Failed to read state file: {error}"))); + return Err(Error::Storage(format!( + "Failed to read state file: {error}" + ))); } }; @@ -288,10 +273,13 @@ mod tests { #[test] fn given_default_storage_type_should_use_file_backend() { + let temp_dir = TempDir::new().expect("tempdir"); let config = StateConfig { enabled: true, storage_type: None, - storage_config: None, + storage_config: Some( + json!({ "base_path": temp_dir.path().to_string_lossy().as_ref() }), + ), state_id: None, }; let storage = create_state_storage(&config).expect("default file storage"); diff --git a/core/integration/tests/connectors/fixtures/mod.rs b/core/integration/tests/connectors/fixtures/mod.rs index d8f18a6dee..34a7da9245 100644 --- a/core/integration/tests/connectors/fixtures/mod.rs +++ b/core/integration/tests/connectors/fixtures/mod.rs @@ -71,7 +71,7 @@ pub use mongodb::{ }; pub use opensearch::{ OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture, - OpenSearchSourceTypedFieldsFixture, + OpenSearchSourceSmallBatchFixture, OpenSearchSourceTypedFieldsFixture, }; pub use postgres::{ PostgresOps, PostgresSinkByteaFixture, PostgresSinkFixture, PostgresSinkJsonFixture, diff --git a/core/integration/tests/connectors/fixtures/opensearch/mod.rs b/core/integration/tests/connectors/fixtures/opensearch/mod.rs index a303d84460..d8a0296ef2 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/mod.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/mod.rs @@ -22,5 +22,5 @@ pub mod source; pub use source::{ OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture, - OpenSearchSourceTypedFieldsFixture, + OpenSearchSourceSmallBatchFixture, OpenSearchSourceTypedFieldsFixture, }; diff --git a/core/integration/tests/connectors/fixtures/opensearch/source.rs b/core/integration/tests/connectors/fixtures/opensearch/source.rs index 66fe0cc0a6..80f51f7b8f 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/source.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/source.rs @@ -234,6 +234,44 @@ impl TestFixture for OpenSearchSourcePreCreatedFixture { } } +/// OpenSearch source fixture with pre-created index and a small `batch_size` for +/// pagination integration tests. +pub struct OpenSearchSourceSmallBatchFixture { + inner: OpenSearchSourceFixture, +} + +impl std::ops::Deref for OpenSearchSourceSmallBatchFixture { + type Target = OpenSearchSourceFixture; + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl OpenSearchOps for OpenSearchSourceSmallBatchFixture { + fn container(&self) -> &OpenSearchContainer { + &self.inner.container + } + + fn http_client(&self) -> &HttpClient { + &self.inner.http_client + } +} + +#[async_trait] +impl TestFixture for OpenSearchSourceSmallBatchFixture { + async fn setup() -> Result { + let inner = OpenSearchSourceFixture::setup().await?; + inner.setup_index().await?; + Ok(Self { inner }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + let mut envs = self.inner.connectors_runtime_envs(); + envs.insert(ENV_SOURCE_BATCH_SIZE.to_string(), "2".to_string()); + envs + } +} + /// OpenSearch source fixture pointing at an index that is never created, /// for exercising the connector's "missing index" failure path. pub struct OpenSearchSourceMissingIndexFixture { diff --git a/core/integration/tests/connectors/opensearch/opensearch_source.rs b/core/integration/tests/connectors/opensearch/opensearch_source.rs index aba5bdfcb9..546751bbfa 100644 --- a/core/integration/tests/connectors/opensearch/opensearch_source.rs +++ b/core/integration/tests/connectors/opensearch/opensearch_source.rs @@ -20,6 +20,7 @@ use super::{POLL_ATTEMPTS, POLL_INTERVAL_MS, TEST_MESSAGE_COUNT}; use crate::connectors::fixtures::{ OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture, + OpenSearchSourceSmallBatchFixture, }; use iggy_common::MessageClient; use iggy_common::{Consumer, Identifier, PollingStrategy}; @@ -150,6 +151,63 @@ async fn given_documents_in_index_when_connector_polls_should_produce_messages( assert_contains_document_ids(&received, &expected_ids); } +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), + seed = seeds::connector_stream +)] +async fn given_more_documents_than_batch_size_when_connector_polls_should_fetch_all( + harness: &TestHarness, + fixture: OpenSearchSourceSmallBatchFixture, +) { + const DOC_COUNT: usize = 6; + + let client = harness.root_client().await.unwrap(); + + fixture + .insert_documents(DOC_COUNT) + .await + .expect("Failed to insert documents"); + + let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); + let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); + let consumer_id: Identifier = "pagination_consumer".try_into().unwrap(); + + let mut received: Vec = Vec::new(); + for _ in 0..POLL_ATTEMPTS { + if let Ok(polled) = client + .poll_messages( + &stream_id, + &topic_id, + None, + &Consumer::new(consumer_id.clone()), + &PollingStrategy::next(), + 10, + true, + ) + .await + { + for msg in polled.messages { + if let Ok(json) = serde_json::from_slice(&msg.payload) { + received.push(json); + } + } + if received.len() >= DOC_COUNT { + break; + } + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + + assert!( + received.len() >= DOC_COUNT, + "Expected at least {DOC_COUNT} messages across search_after pages, got {}", + received.len() + ); + + let expected_ids: Vec = (1..=DOC_COUNT as i64).collect(); + assert_contains_document_ids(&received, &expected_ids); +} + #[iggy_harness( server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), seed = seeds::connector_stream diff --git a/core/integration/tests/connectors/opensearch/opensearch_source_types.rs b/core/integration/tests/connectors/opensearch/opensearch_source_types.rs index 0ff37fa992..74d524ea54 100644 --- a/core/integration/tests/connectors/opensearch/opensearch_source_types.rs +++ b/core/integration/tests/connectors/opensearch/opensearch_source_types.rs @@ -71,7 +71,7 @@ async fn poll_json_messages( server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), seed = seeds::connector_stream )] -async fn opensearch_source_message_payload_structure( +async fn given_document_in_index_when_connector_polls_should_expose_payload_structure( harness: &TestHarness, fixture: OpenSearchSourcePreCreatedFixture, ) { @@ -102,7 +102,7 @@ async fn opensearch_source_message_payload_structure( server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), seed = seeds::connector_stream )] -async fn opensearch_source_typed_fields_should_round_trip_in_payload( +async fn given_typed_fields_document_when_connector_polls_should_round_trip_payload( harness: &TestHarness, fixture: OpenSearchSourceTypedFieldsFixture, ) { @@ -135,7 +135,7 @@ async fn opensearch_source_typed_fields_should_round_trip_in_payload( server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), seed = seeds::connector_stream )] -async fn opensearch_source_search_after_when_second_batch_inserted_should_not_duplicate( +async fn given_first_batch_polled_when_second_batch_inserted_should_not_duplicate( harness: &TestHarness, fixture: OpenSearchSourcePreCreatedFixture, ) { From 536237d960cd3928995e0ea1fc228e61495aa280 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Thu, 18 Jun 2026 11:55:12 -0400 Subject: [PATCH 07/19] Rename fetched->published; fix state/cursor Rename State.total_documents_fetched to total_documents_published across source, state manager, and tests; add serde alias to preserve backwards compat. Change verbose_logging from Option to bool with a default and update tests accordingly. Improve search_after/cursor handling: preserve and restore cursor across empty polls, expose test helpers (test_search_after, test_last_poll_timestamp), and warn when hits lack _source. Harden file state persistence: use compact JSON, perform atomic write with cleanup on error, sync temp file and parent directory, and only save state when appropriate. Add an example opensearch_source.toml and extend integration tests to detect cursor-reset regressions. --- .../connectors/opensearch_source.toml | 54 ++++++++++++ .../opensearch_source/src/http_tests.rs | 60 ++++++++++++- .../sources/opensearch_source/src/lib.rs | 88 ++++++++++++------- .../opensearch_source/src/state_manager.rs | 42 ++++++--- .../opensearch/opensearch_source.rs | 23 +++++ .../opensearch/opensearch_source_types.rs | 78 ++++++++++++---- 6 files changed, 275 insertions(+), 70 deletions(-) create mode 100644 core/connectors/runtime/example_config/connectors/opensearch_source.toml diff --git a/core/connectors/runtime/example_config/connectors/opensearch_source.toml b/core/connectors/runtime/example_config/connectors/opensearch_source.toml new file mode 100644 index 0000000000..b28dac5823 --- /dev/null +++ b/core/connectors/runtime/example_config/connectors/opensearch_source.toml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "source" +key = "opensearch" +enabled = true +version = 0 +name = "OpenSearch source" +path = "/target/release/libiggy_connector_opensearch_source" +plugin_config_format = "json" + +[[streams]] +stream = "opensearch_stream" +topic = "documents" +schema = "json" +batch_length = 100 +linger_time = "5ms" + +[plugin_config] +url = "http://localhost:9200" +index = "logs-*" +polling_interval = "10s" +batch_size = 100 +timestamp_field = "@timestamp" +# username = "admin" +# password = "replace_with_secret" +# verbose_logging = false + +# Optional: restrict which documents are polled (defaults to match_all). +# [plugin_config.query] +# match = { "log.level" = "error" } + +# Optional: mirror state to a local JSON file in addition to runtime msgpack state. +# File state is secondary — runtime ConnectorState (msgpack) is authoritative on restart. +# [plugin_config.state] +# enabled = true +# storage_type = "file" +# state_id = "opensearch_logs_connector" +# [plugin_config.state.storage_config] +# base_path = "./connector_states" diff --git a/core/connectors/sources/opensearch_source/src/http_tests.rs b/core/connectors/sources/opensearch_source/src/http_tests.rs index 7d8b575e47..7fdccf35ba 100644 --- a/core/connectors/sources/opensearch_source/src/http_tests.rs +++ b/core/connectors/sources/opensearch_source/src/http_tests.rs @@ -51,7 +51,7 @@ fn base_config(url: &str) -> OpenSearchSourceConfig { polling_interval: Some("1ms".to_string()), batch_size: Some(10), timestamp_field: Some("timestamp".to_string()), - verbose_logging: None, + verbose_logging: false, state: None, } } @@ -315,6 +315,47 @@ async fn given_hit_without_sort_when_poll_should_skip_document() { let (_, polls, _, empty_polls) = source.test_metrics().await; assert_eq!(polls, 1); assert_eq!(empty_polls, 1); + assert!( + source.test_search_after().await.is_none(), + "sort-missing skip must not corrupt cursor" + ); +} + +#[tokio::test] +async fn given_cursor_set_when_empty_poll_should_preserve_cursor() { + let request_count = Arc::new(AtomicUsize::new(0)); + let count = request_count.clone(); + + let first_page = search_response(vec![search_hit("doc-1", "2024-01-01T00:00:00Z", json!({}))]); + let empty_page = search_response(vec![]); + + let app = mock_router(StatusCode::OK, move |_| { + let first_page = first_page.clone(); + let empty_page = empty_page.clone(); + let count = count.clone(); + async move { + let page = count.fetch_add(1, Ordering::SeqCst); + let response = if page == 0 { first_page } else { empty_page }; + (StatusCode::OK, response.to_string()) + } + }); + let base = start_server(app).await; + let mut source = OpenSearchSource::new(1, base_config(&base), None); + source.open().await.unwrap(); + + source.poll().await.expect("first poll"); + let cursor_after_first = source.test_search_after().await; + assert!( + cursor_after_first.is_some(), + "cursor must be set after non-empty poll" + ); + + source.poll().await.expect("empty poll"); + let cursor_after_empty = source.test_search_after().await; + assert_eq!( + cursor_after_empty, cursor_after_first, + "empty poll must not reset cursor to None" + ); } #[tokio::test] @@ -334,6 +375,10 @@ async fn given_hit_without_source_when_search_should_skip_document() { let produced = source.poll().await.expect("poll should succeed"); assert!(produced.messages.is_empty()); + assert!( + source.test_search_after().await.is_none(), + "_source-missing skip must not corrupt cursor" + ); } #[tokio::test] @@ -384,6 +429,10 @@ async fn given_enabled_file_state_when_open_close_should_persist_state() { let (fetched, polls, _, _) = reloaded.test_metrics().await; assert_eq!(fetched, 1); assert_eq!(polls, 1); + assert!( + reloaded.test_search_after().await.is_some(), + "search_after cursor must be restored from file state" + ); } #[tokio::test] @@ -455,8 +504,11 @@ async fn given_epoch_seconds_timestamp_should_update_last_poll_timestamp() { source.open().await.unwrap(); source.poll().await.unwrap(); - let (_, _, _, _) = source.test_metrics().await; - // Timestamp parsing exercised via poll completing without error. + let last_ts = source.test_last_poll_timestamp().await; + assert!( + last_ts.is_some(), + "epoch-seconds timestamp must be parsed and stored in state" + ); } #[tokio::test] @@ -464,7 +516,7 @@ async fn given_verbose_logging_when_poll_should_succeed() { let app = empty_search_router(StatusCode::OK); let base = start_server(app).await; let mut config = base_config(&base); - config.verbose_logging = Some(true); + config.verbose_logging = true; let mut source = OpenSearchSource::new(1, config, None); source.open().await.unwrap(); source.poll().await.expect("verbose poll should succeed"); diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs index 2a4fe7bf37..52015f7f5c 100644 --- a/core/connectors/sources/opensearch_source/src/lib.rs +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -47,8 +47,8 @@ const DEFAULT_BATCH_SIZE: usize = 100; struct State { #[serde(default)] last_poll_timestamp: Option>, - #[serde(default)] - total_documents_fetched: usize, + #[serde(default, alias = "total_documents_fetched")] + total_documents_published: usize, #[serde(default)] poll_count: usize, /// OpenSearch `search_after` tuple from the last hit in the previous batch. @@ -98,7 +98,8 @@ pub struct OpenSearchSourceConfig { pub polling_interval: Option, pub batch_size: Option, pub timestamp_field: Option, - pub verbose_logging: Option, + #[serde(default)] + pub verbose_logging: bool, pub state: Option, } @@ -132,7 +133,7 @@ impl OpenSearchSource { .query .clone() .unwrap_or_else(|| json!({ "match_all": {} })); - let verbose = config.verbose_logging.unwrap_or(false); + let verbose = config.verbose_logging; let (restored_state, state_restore_error, runtime_state_restored) = restore_state(id, state); @@ -298,6 +299,11 @@ impl OpenSearchSource { }; let Some(source) = hit.get("_source") else { + warn!( + connector_id = self.id, + hit_id = hit.get("_id").and_then(|v| v.as_str()), + "Skipping OpenSearch hit without _source; document will not be published" + ); continue; }; @@ -337,9 +343,11 @@ impl OpenSearchSource { processing_time_ms: f64, ) -> (Vec, Option) { let mut state = self.state.lock().await; - state.total_documents_fetched += outcome.messages.len(); + state.total_documents_published += outcome.messages.len(); state.poll_count += 1; - state.search_after = outcome.search_after; + if let Some(cursor) = outcome.search_after { + state.search_after = Some(cursor); + } if let Some(timestamp) = outcome.last_poll_timestamp { state.last_poll_timestamp = Some(timestamp); } @@ -360,7 +368,7 @@ impl OpenSearchSource { / total_polls as f64; let produced_count = outcome.messages.len(); - let total_documents_fetched = state.total_documents_fetched; + let total_documents_published = state.total_documents_published; let messages = outcome.messages; let persisted_state = self.serialize_state(&state); drop(state); @@ -368,13 +376,13 @@ impl OpenSearchSource { if self.verbose { info!( "OpenSearch source connector ID: {} produced {produced_count} messages. \ - Total fetched: {total_documents_fetched}", + Total published: {total_documents_published}", self.id ); } else { debug!( "OpenSearch source connector ID: {} produced {produced_count} messages. \ - Total fetched: {total_documents_fetched}", + Total published: {total_documents_published}", self.id ); } @@ -391,12 +399,22 @@ impl OpenSearchSource { async fn test_metrics(&self) -> (usize, usize, usize, usize) { let state = self.state.lock().await; ( - state.total_documents_fetched, + state.total_documents_published, state.poll_count, state.error_count, state.processing_stats.empty_polls_count, ) } + + #[cfg(test)] + async fn test_search_after(&self) -> Option> { + self.state.lock().await.search_after.clone() + } + + #[cfg(test)] + async fn test_last_poll_timestamp(&self) -> Option> { + self.state.lock().await.last_poll_timestamp + } } fn restore_state(id: u32, state: Option) -> (State, Option, bool) { @@ -408,8 +426,8 @@ fn restore_state(id: u32, state: Option) -> (State, Option { info!( "Restored state for {CONNECTOR_NAME} connector with ID: {id}. \ - Documents fetched: {}, poll count: {}", - restored.total_documents_fetched, restored.poll_count + Documents published: {}, poll count: {}", + restored.total_documents_published, restored.poll_count ); (restored, None, true) } @@ -542,6 +560,7 @@ impl Source for OpenSearchSource { state.error_count += 1; state.last_error = Some(e.to_string()); drop(state); + sleep(self.polling_interval).await; return Err(e); } }; @@ -558,17 +577,18 @@ impl Source for OpenSearchSource { async fn close(&mut self) -> Result<(), Error> { let state = self.state.lock().await; info!( - "OpenSearch source connector with ID: {} is closing. Stats: {} total documents fetched, {} polls executed, {} errors", - self.id, state.total_documents_fetched, state.poll_count, state.error_count + "OpenSearch source connector with ID: {} is closing. Stats: {} total documents published, {} polls executed, {} errors", + self.id, state.total_documents_published, state.poll_count, state.error_count ); drop(state); - if self - .config - .state - .as_ref() - .map(|s| s.enabled) - .unwrap_or(false) + if self.client.is_some() + && self + .config + .state + .as_ref() + .map(|s| s.enabled) + .unwrap_or(false) { self.save_state().await?; } @@ -599,7 +619,7 @@ mod tests { polling_interval: Some("100ms".to_string()), batch_size: Some(10), timestamp_field: Some("timestamp".to_string()), - verbose_logging: None, + verbose_logging: false, state: None, } } @@ -607,8 +627,8 @@ mod tests { fn test_state() -> State { State { last_poll_timestamp: None, - total_documents_fetched: 500, - poll_count: 5, + total_documents_published: 500, + poll_count: 7, search_after: Some(vec![json!("2024-01-01T00:00:00Z"), json!("doc_42")]), error_count: 1, last_error: Some("connection reset".to_string()), @@ -623,7 +643,7 @@ mod tests { } #[test] - fn given_persisted_state_should_restore_total_documents_fetched() { + fn given_persisted_state_should_restore_total_documents_published() { let state = test_state(); let serialized = rmp_serde::to_vec(&state).expect("Failed to serialize state"); let connector_state = ConnectorState(serialized); @@ -633,8 +653,8 @@ mod tests { let runtime = tokio::runtime::Runtime::new().unwrap(); runtime.block_on(async { let restored = source.state.lock().await; - assert_eq!(restored.total_documents_fetched, 500); - assert_eq!(restored.poll_count, 5); + assert_eq!(restored.total_documents_published, 500); + assert_eq!(restored.poll_count, 7); assert_eq!( restored.search_after, Some(vec![json!("2024-01-01T00:00:00Z"), json!("doc_42")]) @@ -651,7 +671,7 @@ mod tests { let runtime = tokio::runtime::Runtime::new().unwrap(); runtime.block_on(async { let state = source.state.lock().await; - assert_eq!(state.total_documents_fetched, 0); + assert_eq!(state.total_documents_published, 0); assert_eq!(state.poll_count, 0); assert_eq!(state.search_after, None); assert!(source.state_restore_error.is_none()); @@ -668,7 +688,7 @@ mod tests { let runtime = tokio::runtime::Runtime::new().unwrap(); runtime.block_on(async { let state = source.state.lock().await; - assert_eq!(state.total_documents_fetched, 0); + assert_eq!(state.total_documents_published, 0); assert_eq!(state.poll_count, 0); }); } @@ -719,7 +739,7 @@ mod tests { last_updated: Utc::now(), version: SOURCE_STATE_VERSION, data: json!({ - "total_documents_fetched": 9, + "total_documents_published": 9, "poll_count": 4, "search_after": ["2024-02-01T00:00:00Z", "doc-9"], "error_count": 2, @@ -741,7 +761,7 @@ mod tests { .expect("apply source state"); let state = source.state.lock().await; - assert_eq!(state.total_documents_fetched, 9); + assert_eq!(state.total_documents_published, 9); assert_eq!(state.poll_count, 4); assert_eq!( state.search_after, @@ -800,7 +820,7 @@ mod tests { .await .expect("export state"); assert_eq!(exported.id, "opensearch_source_1"); - assert_eq!(exported.data["total_documents_fetched"], 500); + assert_eq!(exported.data["total_documents_published"], 500); assert_eq!( exported.metadata.as_ref().unwrap()["index"], "test_documents" @@ -837,8 +857,8 @@ mod tests { rmp_serde::from_slice(&serialized).expect("Failed to deserialize"); assert_eq!( - original.total_documents_fetched, - deserialized.total_documents_fetched + original.total_documents_published, + deserialized.total_documents_published ); assert_eq!(original.poll_count, deserialized.poll_count); assert_eq!(original.search_after, deserialized.search_after); @@ -857,6 +877,6 @@ mod tests { .unwrap() .deserialize(CONNECTOR_NAME, 1) .expect("Failed to deserialize state"); - assert_eq!(restored.total_documents_fetched, 500); + assert_eq!(restored.total_documents_published, 500); } } diff --git a/core/connectors/sources/opensearch_source/src/state_manager.rs b/core/connectors/sources/opensearch_source/src/state_manager.rs index 72b98a0d5b..65e28d30d0 100644 --- a/core/connectors/sources/opensearch_source/src/state_manager.rs +++ b/core/connectors/sources/opensearch_source/src/state_manager.rs @@ -55,17 +55,17 @@ impl OpenSearchSource { if let Some(source_state) = storage.load_source_state(&state_id).await? { self.source_state_to_internal_state(source_state).await?; - let (last_poll_timestamp, total_documents_fetched, poll_count) = { + let (last_poll_timestamp, total_documents_published, poll_count) = { let state = self.state.lock().await; ( state.last_poll_timestamp, - state.total_documents_fetched, + state.total_documents_published, state.poll_count, ) }; info!( "Loaded state for OpenSearch source connector with ID: {} - last poll: {:?}, total docs: {}, polls: {}", - self.id, last_poll_timestamp, total_documents_fetched, poll_count + self.id, last_poll_timestamp, total_documents_published, poll_count ); } else { info!( @@ -145,7 +145,7 @@ impl StateStorage for FileStateStorage { let path = self.get_state_path(&state.id); let tmp_path = path.with_extension("json.tmp"); - let json = serde_json::to_string_pretty(state) + let json = serde_json::to_string(state) .map_err(|e| Error::Serialization(format!("Failed to serialize source state: {e}")))?; let mut tmp_file = OpenOptions::new() @@ -155,20 +155,34 @@ impl StateStorage for FileStateStorage { .open(&tmp_path) .await .map_err(|e| Error::Storage(format!("Failed to open state temp file: {e}")))?; - tmp_file - .write_all(json.as_bytes()) - .await - .map_err(|e| Error::Storage(format!("Failed to write state temp file: {e}")))?; - tmp_file - .sync_data() - .await - .map_err(|e| Error::Storage(format!("Failed to sync state temp file: {e}")))?; + if let Err(e) = tmp_file.write_all(json.as_bytes()).await { + let _ = fs::remove_file(&tmp_path).await; + return Err(Error::Storage(format!( + "Failed to write state temp file: {e}" + ))); + } + if let Err(e) = tmp_file.sync_data().await { + let _ = fs::remove_file(&tmp_path).await; + return Err(Error::Storage(format!( + "Failed to sync state temp file: {e}" + ))); + } drop(tmp_file); fs::rename(&tmp_path, &path) .await .map_err(|e| Error::Storage(format!("Failed to rename state file: {e}")))?; + // Flush the parent directory entry so the rename is durable on crash. + if let Some(parent) = path.parent() { + let dir = tokio::fs::File::open(parent) + .await + .map_err(|e| Error::Storage(format!("Failed to open state directory: {e}")))?; + dir.sync_all() + .await + .map_err(|e| Error::Storage(format!("Failed to sync state directory: {e}")))?; + } + Ok(()) } @@ -235,7 +249,7 @@ mod tests { last_updated: Utc::now(), version: SOURCE_STATE_VERSION, data: json!({ - "total_documents_fetched": 7, + "total_documents_published": 7, "poll_count": 2, "search_after": ["2024-01-01T00:00:00Z", "doc-7"] }), @@ -251,7 +265,7 @@ mod tests { .await .expect("load state") .expect("state file should exist"); - assert_eq!(loaded.data["total_documents_fetched"], 7); + assert_eq!(loaded.data["total_documents_published"], 7); assert_eq!(loaded.data["poll_count"], 2); }); } diff --git a/core/integration/tests/connectors/opensearch/opensearch_source.rs b/core/integration/tests/connectors/opensearch/opensearch_source.rs index 546751bbfa..2da0ed8545 100644 --- a/core/integration/tests/connectors/opensearch/opensearch_source.rs +++ b/core/integration/tests/connectors/opensearch/opensearch_source.rs @@ -206,6 +206,29 @@ async fn given_more_documents_than_batch_size_when_connector_polls_should_fetch_ let expected_ids: Vec = (1..=DOC_COUNT as i64).collect(); assert_contains_document_ids(&received, &expected_ids); + + // Wait for at least one empty poll to fire (connector catches up to end of index). + // A cursor-reset bug would cause the connector to re-fetch all docs on the next empty poll. + sleep(Duration::from_millis(POLL_INTERVAL_MS * 5)).await; + + let audit_consumer: Identifier = "pagination_audit".try_into().unwrap(); + let all_on_stream = + poll_all_messages_from_offset_zero(&client, &audit_consumer, DOC_COUNT).await; + + let all_ids: Vec = all_on_stream + .iter() + .filter_map(|record| record.get("id").and_then(|v| v.as_i64())) + .collect(); + let unique_count = document_ids(&all_on_stream).len(); + assert_eq!( + all_ids.len(), + unique_count, + "stream contains duplicate document IDs after empty poll; cursor was reset" + ); + assert_eq!( + unique_count, DOC_COUNT, + "expected exactly {DOC_COUNT} unique documents on stream, got {unique_count}" + ); } #[iggy_harness( diff --git a/core/integration/tests/connectors/opensearch/opensearch_source_types.rs b/core/integration/tests/connectors/opensearch/opensearch_source_types.rs index 74d524ea54..1848137ae7 100644 --- a/core/integration/tests/connectors/opensearch/opensearch_source_types.rs +++ b/core/integration/tests/connectors/opensearch/opensearch_source_types.rs @@ -167,31 +167,73 @@ async fn given_first_batch_polled_when_second_batch_inserted_should_not_duplicat } fixture.refresh_index().await.expect("refresh index"); - let mut second_batch = Vec::new(); + // Wait for second batch to arrive, collecting only new IDs. + let mut second_batch_seen = false; for _ in 0..POLL_ATTEMPTS { let polled = poll_json_messages(&client, &consumer_id, 10).await; - for record in polled { - let id = record.get("id").and_then(Value::as_i64).unwrap_or(0); - if !first_ids.contains(&id) { - second_batch.push(record); - } - } - if second_batch.len() >= TEST_MESSAGE_COUNT { + let new_count = polled + .iter() + .filter_map(|r| r.get("id").and_then(Value::as_i64)) + .filter(|id| !first_ids.contains(id)) + .count(); + if new_count >= TEST_MESSAGE_COUNT { + second_batch_seen = true; break; } sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; } + assert!(second_batch_seen, "second batch never arrived"); + + // Verify the full stream from offset 0 contains exactly 2*TEST_MESSAGE_COUNT unique docs. + // If the cursor reset bug were present, the connector would re-emit first-batch docs and + // the stream would contain duplicates. + let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); + let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); + let audit_consumer: Identifier = "no_dup_audit".try_into().unwrap(); + let mut all_on_stream: Vec = Vec::new(); + for _ in 0..POLL_ATTEMPTS { + if let Ok(polled) = client + .poll_messages( + &stream_id, + &topic_id, + None, + &Consumer::new(audit_consumer.clone()), + &PollingStrategy::offset(0), + 100, + false, + ) + .await + { + all_on_stream.clear(); + for msg in polled.messages { + if let Ok(json) = serde_json::from_slice::(&msg.payload) { + all_on_stream.push(json); + } + } + if all_on_stream.len() >= TEST_MESSAGE_COUNT * 2 { + break; + } + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + let all_ids: Vec = all_on_stream + .iter() + .filter_map(|r| r.get("id").and_then(Value::as_i64)) + .collect(); + let unique_ids: HashSet = all_ids.iter().copied().collect(); assert_eq!( - second_batch.len(), - TEST_MESSAGE_COUNT, - "expected second batch only" + all_ids.len(), + unique_ids.len(), + "stream has {} total IDs but only {} unique; cursor reset caused re-delivery", + all_ids.len(), + unique_ids.len() + ); + assert_eq!( + unique_ids.len(), + TEST_MESSAGE_COUNT * 2, + "expected {} unique docs on stream (first + second batch), got {}", + TEST_MESSAGE_COUNT * 2, + unique_ids.len() ); - for record in &second_batch { - let id = record.get("id").and_then(Value::as_i64).unwrap_or(0); - assert!( - id > TEST_MESSAGE_COUNT as i64, - "duplicate or first-batch id in second batch: {id}" - ); - } } From aff88cde840d15dda7d66a938d8a28fcb0e942df Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Thu, 18 Jun 2026 12:28:29 -0400 Subject: [PATCH 08/19] Fix test function name spelling Rename test function from `given_unparseable_timestamp_value_should_return_none` to `given_unparsable_timestamp_value_should_return_none` to correct spelling/consistency in the tests. This is a non-functional change and does not affect runtime behavior. --- core/connectors/sources/opensearch_source/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs index 52015f7f5c..c7845a605e 100644 --- a/core/connectors/sources/opensearch_source/src/lib.rs +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -722,7 +722,7 @@ mod tests { } #[test] - fn given_unparseable_timestamp_value_should_return_none() { + fn given_unparsable_timestamp_value_should_return_none() { let value = json!("not-a-timestamp"); assert!(parse_document_timestamp(&value).is_none()); } From 9a626781e0d7393cd40b55122bcb500807a068f8 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Thu, 18 Jun 2026 13:13:45 -0400 Subject: [PATCH 09/19] Update README with architecture Rewrite and expand the OpenSearch source README: replace brief Features section with an Architecture overview, add detailed "How it works" (poll cycle, search request, per-hit processing, timestamp parsing), and document state & persistence (runtime msgpack + optional file mirror). Clarify config schema with types/defaults, change default polling_interval to "10s", and add requirements, error variants, limitations, throughput tuning, and troubleshooting guidance. Also standardize tables and behavior notes (search_after semantics, cursor invariants, file save atomicity, at-least-once delivery, missing _source handling). --- .../sources/opensearch_source/README.md | 229 +++++++++++++++--- 1 file changed, 195 insertions(+), 34 deletions(-) diff --git a/core/connectors/sources/opensearch_source/README.md b/core/connectors/sources/opensearch_source/README.md index f2c19a9f6f..00f21e2b6f 100644 --- a/core/connectors/sources/opensearch_source/README.md +++ b/core/connectors/sources/opensearch_source/README.md @@ -4,13 +4,18 @@ Polls documents from an OpenSearch index and publishes them to Iggy streams as J messages. Incremental progress is tracked with OpenSearch `search_after` pagination on `(timestamp_field, _id)`. -## Features +## Architecture -- Incremental polling via `search_after` on a configured timestamp field plus `_id` -- Optional custom OpenSearch query (`match_all` by default) -- Basic authentication (username + password) -- Runtime state persistence via the connectors runtime (`ConnectorState` / MessagePack) -- Optional supplementary file-backed state when `plugin_config.state.enabled = true` +The connector is a cdylib source plugin loaded by the Iggy connectors runtime via FFI. + +| Layer | Crate / binary | Role | +| ----- | -------------- | ---- | +| Plugin | `iggy_connector_opensearch_source` | Implements `Source` trait; talks to OpenSearch | +| SDK | `iggy_connector_sdk` | `Source`, `ProducedMessage`, `ConnectorState`, `source_connector!` macro | +| Runtime | `iggy-connectors` | Loads `.dylib`, calls `open` / `poll` / `close`, publishes to Iggy, saves `ConnectorState` | +| Server | `iggy-server` | Receives messages on configured streams/topics | + +The connector is read-only. It does not write to OpenSearch. ## Configuration @@ -33,7 +38,7 @@ linger_time = "5ms" [plugin_config] url = "http://localhost:9200" index = "logs-*" -polling_interval = "30s" +polling_interval = "10s" batch_size = 100 timestamp_field = "@timestamp" query = { "match_all": {} } @@ -41,21 +46,21 @@ query = { "match_all": {} } ### Required fields -| Field | Description | -| --- | --- | -| `url` | OpenSearch HTTP endpoint | -| `index` | Index name or pattern | -| `timestamp_field` | Document field used for sort order and incremental cursors (required) | +| Field | Type | Description | +| ----- | ---- | ----------- | +| `url` | `String` | OpenSearch HTTP base URL | +| `index` | `String` | Index name or pattern | +| `timestamp_field` | `String` | Document field used for sort order and cursor; must exist on every document | ### Optional fields -| Field | Default | Description | -| --- | --- | --- | -| `polling_interval` | `10s` | Delay before each poll (humantime format) | -| `batch_size` | `100` | Maximum documents per search request (minimum `1`) | -| `query` | `match_all` | OpenSearch query DSL object | -| `username` / `password` | none | HTTP basic authentication | -| `verbose_logging` | `false` | Emit per-poll batch counts at `info!` instead of `debug!` | +| Field | Type | Default | Description | +| ----- | ---- | ------- | ----------- | +| `polling_interval` | `String` | `"10s"` | Delay after each completed poll cycle (humantime format). First poll runs immediately. | +| `batch_size` | `usize` | `100` | Documents per search request (minimum `1`) | +| `query` | JSON object | `{"match_all": {}}` | OpenSearch query DSL; applied on every poll | +| `username` / `password` | `String` | none | HTTP basic authentication | +| `verbose_logging` | `bool` | `false` | Log per-poll batch counts at `info!` instead of `debug!` | ### File-backed state (optional) @@ -70,31 +75,187 @@ storage_config = { base_path = "./connector_states" } state_id = "opensearch_logs_connector" ``` -Only `storage_type = "file"` is implemented. State is saved on `close()` when -`state.enabled = true`. +Only `storage_type = "file"` is implemented. See [State and persistence](#state-and-persistence). + +## How it works + +### Poll cycle + +Each call to `poll()`: + +1. Issues `POST /{index}/_search` with the query below. +2. Maps each hit's `_source` to a JSON `ProducedMessage`. +3. Updates the `search_after` cursor to the sort tuple of the last successfully published hit. +4. Returns `ProducedMessages` containing the messages and a serialized `ConnectorState`. +5. Sleeps `polling_interval` before returning. + +The runtime persists `ConnectorState` (msgpack) after each successful `poll()` return. + +### Search request + +```json +{ + "query": "", + "size": "", + "sort": [ + { "": { "order": "asc" } }, + { "_id": { "order": "asc" } } + ], + "search_after": [""] +} +``` + +Two sort keys give stable order when timestamps collide. `_id` is the tiebreaker. + +### Per-hit processing + +For each hit in the response: + +1. **Missing sort tuple** — skip with `warn!`; cursor not advanced. +2. **Missing `_source`** — skip with `warn!`; cursor not advanced. +3. Both present — serialize `_source` as JSON payload; advance cursor to this hit's sort tuple. + +The cursor (`search_after`) only advances for hits where **both** sort and `_source` are present. +An empty batch leaves the cursor unchanged. + +### Timestamp parsing + +The `timestamp_field` value in `_source` is parsed to populate `last_poll_timestamp` (informational only; does not affect pagination). + +| `_source` value | Parsing | +| --------------- | ------- | +| RFC 3339 string | `DateTime::parse_from_rfc3339` | +| Integer `> 1e12` | Epoch milliseconds | +| Integer `≤ 1e12` | Epoch seconds | +| Other | Ignored; document still published | + +## State and persistence + +### Internal state fields + +| Field | Purpose | +| ----- | ------- | +| `search_after` | `Option>` — OpenSearch sort tuple from last published hit; authoritative resume cursor | +| `last_poll_timestamp` | `Option>` — timestamp of last processed document; informational | +| `total_documents_published` | Cumulative documents emitted to Iggy | +| `poll_count` | Total search requests executed (successful + empty) | +| `error_count` / `last_error` | Search failure tracking | +| `processing_stats` | Bytes processed, empty/successful poll counts, avg latency | + +**Invariant:** `search_after` is the authoritative resume cursor. `last_poll_timestamp` is +informational only and does not affect pagination. + +### Dual persistence + +| Mechanism | Format | When written | When read | Failure mode | +| --------- | ------ | ------------ | --------- | ------------ | +| Runtime `ConnectorState` | MessagePack | Every `poll()` return | `new(id, config, Some(state))` | Corrupt → `open()` fails with `InitError` | +| File `SourceState` | JSON | `close()` if `state.enabled` and connector opened successfully | `open()` if `state.enabled` and no runtime state present | Load failure → `open()` fails with `InitError` | + +File path: `{base_path}/{state_id}.json`; defaults: `base_path = "./connector_states"`, +`state_id = "opensearch_source_{id}"`. + +Runtime `ConnectorState` is authoritative. When valid runtime state is restored on +startup, file state is not loaded. File mirror is written atomically (write-tmp → +fdatasync → rename → dir-fsync) on `close()`. + +## Initial load and tuning + +### Cursor behavior by phase + +| Phase | Cursor behavior | +| ----- | --------------- | +| Fresh start (no state) | No `search_after` — reads from start of sort order | +| Steady state | `search_after` advances — only documents after cursor returned | +| Restart with saved state | Cursor restored from `ConnectorState`; resumes without re-reading | + +There is no separate initial-load code path. Every poll uses the same logic. + +### Throughput + +With defaults (`batch_size = 100`, `polling_interval = "10s"`): + +```text +100 docs / 10s ≈ 10 docs/sec +10,000,000 docs ≈ ~11.5 days to catch up +``` + +Aggressive config for large initial loads: + +```toml +[plugin_config] +polling_interval = "100ms" +batch_size = 5000 +timestamp_field = "@timestamp" +``` + +Optional time-window queries for manual partitioning: + +```toml +[plugin_config] +query = { "range" = { "@timestamp" = { "gte" = "2024-01-01", "lt" = "2024-02-01" } } } +``` -## State fields +Requirements for correct operation: -The connector tracks: +- `timestamp_field` present on every document. +- Index mapping has a date-type field for `timestamp_field`. +- `_source` enabled in the index mapping (see Limitations). -- `search_after`: OpenSearch sort tuple from the last document in the previous batch -- `last_poll_timestamp`: timestamp of the last processed document -- `last_document_id`: `_id` of the last processed document -- `total_documents_fetched`, `poll_count`, error counters, and processing statistics +## Error handling -Corrupt runtime state causes `open()` to fail with `InitError` rather than silently -resetting the cursor. +| Error variant | When raised | +| ------------- | ----------- | +| `InitError` | Corrupt runtime state; missing index at `open()`; file state load failure | +| `InvalidConfigValue` | Missing `timestamp_field`; `batch_size = 0`; unsupported `storage_type` | +| `Storage` | Network or HTTP errors; client not initialized at `poll()` | +| `Serialization` | JSON / MessagePack failures | -## Timestamp formats +## Limitations -The configured `timestamp_field` may be an RFC 3339 string or epoch milliseconds / -seconds in the document `_source`. +- **Single sequential reader** — one `search_after` cursor, one batch per poll. + No parallel shard/slice workers or dedicated bulk-ingest mode. +- **Same path for initial load and steady state** — a fresh connector walks the + index from the oldest `(timestamp_field, _id)` upward. There is no separate + bootstrap implementation. +- **Throughput tied to `polling_interval` and `batch_size`** — defaults (`10s`, + `100`) yield roughly 10 documents/second. Tens of millions of documents require + tuning both knobs and sufficient OpenSearch / Iggy capacity. +- **`search_after` only** — no Scroll API, point-in-time (PIT), or sliced + parallel export. Offset paging (`from`/`size`) is not used. +- **At-least-once delivery** — no deduplication by `_id`. The in-memory cursor advances + before the runtime persists `ConnectorState`; a crash can re-emit the last batch. +- **No HTTP retry** — transient OpenSearch errors fail the poll immediately. No circuit + breaker (unlike the InfluxDB source). +- **Backfill gap** — documents indexed with `timestamp_field` values older than + the current cursor are not read until connector state is reset. +- **Full `_source` only** — entire document JSON is published; no field + projection or schema variants beyond `Schema::Json`. +- **Optional file state** — only `storage_type = "file"` is implemented. File mirror is + written atomically on `close()`, not every poll. Runtime msgpack wins on restart when + both are present. A failed file save on `close()` returns an error. +- **`_source`-disabled documents skipped permanently** — hits returned without `_source` + (e.g., index mapping with `"_source": false`) are skipped with a `warn!`. The cursor + advances past them. If `_source` later becomes available for a document at the same + `(timestamp_field, _id)` sort position, it will not be re-fetched. Ensure `_source` is + enabled in the index mapping before using this connector. +- **Missing sort tuple causes no cursor advance** — hits returned without a sort tuple + (rare; typically deleted-doc artifacts or partial shard results) are skipped with a + `warn!`. If such hits appear at the tail of a batch, the cursor stays at the last + successfully published document. Subsequent polls return the same hits until OpenSearch + stops including them. +- **Single-node transport** — `SingleNodeConnectionPool` to `url`; no cluster + node sniffing. ## Troubleshooting | Symptom | Check | -| --- | --- | +| ------- | ----- | | `open()` fails with missing index | Index name, URL, and credentials | | `open()` fails with `state restore failed` | Delete or repair the connector runtime state file | +| `open()` fails with `file state load failed` | Delete or repair the file state JSON | | No new documents after restart | `timestamp_field` mapping must match indexed documents | -| Duplicate messages | Lower `batch_size` only after confirming sort stability on `(timestamp_field, _id)` | +| Duplicate messages | At-least-once delivery; lower `batch_size` only after confirming sort stability on `(timestamp_field, _id)` | +| Initial load too slow | Increase `batch_size`, decrease `polling_interval` | +| Backfilled docs missing | Timestamps older than cursor are skipped; reset state or adjust query | +| Repeated `warn!` about missing `_source` | Index mapping has `"_source": false`; connector cannot publish those documents | From de8faae2ec51abdc47335fa8438e8b6503120307 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Thu, 18 Jun 2026 16:43:55 -0400 Subject: [PATCH 10/19] Error on batches lacking sort values Change OpenSearch source behavior to treat a batch where none of the hits include a sort tuple as an error and avoid advancing the cursor. In lib.rs last_sort is captured earlier and an explicit Storage error is returned when hits are present but no hit provided a sort tuple. Tests in http_tests.rs were updated: the previous "skip when sort missing" test was renamed and now expects poll() to return an error and increment the error metric; additional test verifies that a trailing hit missing _source still advances the cursor past that hit so it won't be re-fetched. These changes prevent corrupting the search_after cursor for all-no-sort batches while preserving correct advancement for batches that contain missing _source entries. --- .../opensearch_source/src/http_tests.rs | 71 ++++++++++++++++--- .../sources/opensearch_source/src/lib.rs | 12 +++- 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/core/connectors/sources/opensearch_source/src/http_tests.rs b/core/connectors/sources/opensearch_source/src/http_tests.rs index 7fdccf35ba..5f564a4e3c 100644 --- a/core/connectors/sources/opensearch_source/src/http_tests.rs +++ b/core/connectors/sources/opensearch_source/src/http_tests.rs @@ -295,7 +295,7 @@ async fn given_basic_auth_when_search_should_send_authorization_header() { } #[tokio::test] -async fn given_hit_without_sort_when_poll_should_skip_document() { +async fn given_batch_where_all_hits_lack_sort_when_poll_should_return_error() { let hit = json!({ "_id": "doc-1", "_source": { "id": 1, "timestamp": "2024-01-01T00:00:00Z" } @@ -309,15 +309,17 @@ async fn given_hit_without_sort_when_poll_should_skip_document() { let mut source = OpenSearchSource::new(1, base_config(&base), None); source.open().await.unwrap(); - let produced = source.poll().await.expect("poll should succeed"); - assert!(produced.messages.is_empty()); + let error = source.poll().await.expect_err("all-no-sort batch must fail"); + assert!( + matches!(error, Error::Storage(_)), + "expected Storage error, got {error:?}" + ); - let (_, polls, _, empty_polls) = source.test_metrics().await; - assert_eq!(polls, 1); - assert_eq!(empty_polls, 1); + let (_, _, errors, _) = source.test_metrics().await; + assert_eq!(errors, 1, "error counter must be incremented"); assert!( source.test_search_after().await.is_none(), - "sort-missing skip must not corrupt cursor" + "cursor must not advance when batch errors" ); } @@ -376,8 +378,59 @@ async fn given_hit_without_source_when_search_should_skip_document() { let produced = source.poll().await.expect("poll should succeed"); assert!(produced.messages.is_empty()); assert!( - source.test_search_after().await.is_none(), - "_source-missing skip must not corrupt cursor" + source.test_search_after().await.is_some(), + "_source-missing skip must still advance cursor to that hit's sort position" + ); +} + +#[tokio::test] +async fn given_trailing_hit_without_source_when_poll_should_advance_cursor_past_it() { + let request_count = Arc::new(AtomicUsize::new(0)); + let count = request_count.clone(); + + let first_page = search_response(vec![ + search_hit("doc-1", "2024-01-01T00:00:00Z", json!({})), + json!({ + "_id": "doc-2", + "sort": ["2024-01-02T00:00:00Z", "doc-2"] + }), + ]); + let empty_page = search_response(vec![]); + + let app = mock_router(StatusCode::OK, move |_| { + let first_page = first_page.clone(); + let empty_page = empty_page.clone(); + let count = count.clone(); + async move { + let page = count.fetch_add(1, Ordering::SeqCst); + let response = if page == 0 { first_page } else { empty_page }; + (StatusCode::OK, response.to_string()) + } + }); + let base = start_server(app).await; + let mut source = OpenSearchSource::new(1, base_config(&base), None); + source.open().await.unwrap(); + + let produced = source.poll().await.expect("first poll"); + assert_eq!(produced.messages.len(), 1, "only doc-1 published; doc-2 has no _source"); + + let cursor = source.test_search_after().await; + assert!( + cursor.is_some(), + "cursor must be set after batch with trailing no-_source hit" + ); + let cursor_vals = cursor.unwrap(); + assert_eq!( + cursor_vals[1].as_str(), + Some("doc-2"), + "cursor must point to doc-2 (trailing no-_source), not doc-1" + ); + + // Second poll must get empty page (cursor past doc-2), not re-fetch doc-2. + let produced2 = source.poll().await.expect("second poll"); + assert!( + produced2.messages.is_empty(), + "doc-2 (no _source) must not be re-fetched after cursor advances past it" ); } diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs index c7845a605e..293d6d6747 100644 --- a/core/connectors/sources/opensearch_source/src/lib.rs +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -298,6 +298,8 @@ impl OpenSearchSource { continue; }; + last_sort = Some(sort); + let Some(source) = hit.get("_source") else { warn!( connector_id = self.id, @@ -307,8 +309,6 @@ impl OpenSearchSource { continue; }; - last_sort = Some(sort); - if let Some(timestamp_value) = source.get(timestamp_field) && let Some(timestamp_utc) = parse_document_timestamp(timestamp_value) { @@ -329,6 +329,14 @@ impl OpenSearchSource { }); } + if !hits.is_empty() && last_sort.is_none() { + return Err(Error::Storage(format!( + "OpenSearch returned {} hit(s) but none had a sort tuple; \ + index may be missing the sort field or using an incompatible mapping", + hits.len() + ))); + } + Ok(SearchOutcome { messages, search_after: last_sort.map(ToOwned::to_owned), From 3dd220b5ffddf4ec8bdfb6f57b1ed6bf82ef1f23 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Thu, 18 Jun 2026 17:08:50 -0400 Subject: [PATCH 11/19] Ignore empty sort and clean temp on rename error Avoid treating an empty 'sort' array as valid by filtering out empty arrays when extracting the sort value from hits (core/connectors/sources/opensearch_source/src/lib.rs). In the state storage rename path (core/connectors/sources/opensearch_source/src/state_manager.rs), attempt to remove the temporary file if fs::rename fails and return a Storage error, preventing leftover temp files on failure. --- core/connectors/sources/opensearch_source/src/lib.rs | 2 +- .../sources/opensearch_source/src/state_manager.rs | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs index 293d6d6747..9ebd937720 100644 --- a/core/connectors/sources/opensearch_source/src/lib.rs +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -289,7 +289,7 @@ impl OpenSearchSource { let mut last_poll_timestamp = None; for hit in &hits { - let Some(sort) = hit.get("sort").and_then(|s| s.as_array()) else { + let Some(sort) = hit.get("sort").and_then(|s| s.as_array()).filter(|a| !a.is_empty()) else { warn!( connector_id = self.id, hit_id = hit.get("_id").and_then(|value| value.as_str()), diff --git a/core/connectors/sources/opensearch_source/src/state_manager.rs b/core/connectors/sources/opensearch_source/src/state_manager.rs index 65e28d30d0..994735e6cf 100644 --- a/core/connectors/sources/opensearch_source/src/state_manager.rs +++ b/core/connectors/sources/opensearch_source/src/state_manager.rs @@ -169,9 +169,10 @@ impl StateStorage for FileStateStorage { } drop(tmp_file); - fs::rename(&tmp_path, &path) - .await - .map_err(|e| Error::Storage(format!("Failed to rename state file: {e}")))?; + if let Err(e) = fs::rename(&tmp_path, &path).await { + let _ = fs::remove_file(&tmp_path).await; + return Err(Error::Storage(format!("Failed to rename state file: {e}"))); + } // Flush the parent directory entry so the rename is durable on crash. if let Some(parent) = path.parent() { From 9f79493eba89470f0a3171d562d3c8d99a7bec77 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Fri, 19 Jun 2026 08:43:25 -0400 Subject: [PATCH 12/19] opensearch source: add retry & circuit breaker Implement HTTP retry and circuit-breaker resilience for the OpenSearch source. Adds configurable plugin options (max_retries, retry_delay, retry_max_delay, max_open_retries, open_retry_max_delay, circuit_breaker_threshold, circuit_breaker_cool_down) and example config updates. Introduces a new retry helper module (retry.rs) with backoff/jitter/retry-after handling and integrates retry logic into index-exists and search calls in lib.rs (send_search_with_retry, check_index_exists_with_retry). Adds circuit breaker usage (recording successes/failures and skipping polls while open) and refactors poll/error handling. Documents behavior in docs/RESILIENCE.md and expands README with semantics and usage notes (including cursor advancement rules for missing sort/_source). Adds unit tests (http_tests.rs) and integration fixtures/tests (wiremock-backed resilience fixtures and runtime tests), plus connector test configs.. Some of the files are transient. They will be cleaned up after all known issues are fixed. --- .../connectors/opensearch_source.toml | 9 + .../sources/opensearch_source/README.md | 33 +- .../opensearch_source/docs/RESILIENCE.md | 176 ++++++++++ .../opensearch_source/src/http_tests.rs | 73 ++++- .../sources/opensearch_source/src/lib.rs | 310 +++++++++++++++--- .../sources/opensearch_source/src/retry.rs | 93 ++++++ .../tests/connectors/fixtures/mod.rs | 5 +- .../fixtures/opensearch/container.rs | 14 + .../connectors/fixtures/opensearch/mod.rs | 4 + .../fixtures/opensearch/resilience.rs | 195 +++++++++++ .../tests/connectors/opensearch/mod.rs | 1 + .../opensearch_source_resilience.rs | 167 ++++++++++ .../opensearch/plugin_config/config.toml | 45 +++ .../opensearch/source_resilience.toml | 20 ++ 14 files changed, 1077 insertions(+), 68 deletions(-) create mode 100644 core/connectors/sources/opensearch_source/docs/RESILIENCE.md create mode 100644 core/connectors/sources/opensearch_source/src/retry.rs create mode 100644 core/integration/tests/connectors/fixtures/opensearch/resilience.rs create mode 100644 core/integration/tests/connectors/opensearch/opensearch_source_resilience.rs create mode 100644 core/integration/tests/connectors/opensearch/plugin_config/config.toml create mode 100644 core/integration/tests/connectors/opensearch/source_resilience.toml diff --git a/core/connectors/runtime/example_config/connectors/opensearch_source.toml b/core/connectors/runtime/example_config/connectors/opensearch_source.toml index b28dac5823..8906cc7ede 100644 --- a/core/connectors/runtime/example_config/connectors/opensearch_source.toml +++ b/core/connectors/runtime/example_config/connectors/opensearch_source.toml @@ -40,6 +40,15 @@ timestamp_field = "@timestamp" # password = "replace_with_secret" # verbose_logging = false +# HTTP resilience (defaults shown; see docs/RESILIENCE.md). +# max_retries = 3 +# retry_delay = "1s" +# retry_max_delay = "30s" +# max_open_retries = 5 +# open_retry_max_delay = "30s" +# circuit_breaker_threshold = 5 +# circuit_breaker_cool_down = "60s" + # Optional: restrict which documents are polled (defaults to match_all). # [plugin_config.query] # match = { "log.level" = "error" } diff --git a/core/connectors/sources/opensearch_source/README.md b/core/connectors/sources/opensearch_source/README.md index 00f21e2b6f..2b27f4a1c0 100644 --- a/core/connectors/sources/opensearch_source/README.md +++ b/core/connectors/sources/opensearch_source/README.md @@ -61,6 +61,15 @@ query = { "match_all": {} } | `query` | JSON object | `{"match_all": {}}` | OpenSearch query DSL; applied on every poll | | `username` / `password` | `String` | none | HTTP basic authentication | | `verbose_logging` | `bool` | `false` | Log per-poll batch counts at `info!` instead of `debug!` | +| `max_retries` | `u32` | `3` | Total HTTP attempts per search during `poll()` | +| `retry_delay` | `String` | `"1s"` | Base backoff between HTTP retries | +| `retry_max_delay` | `String` | `"30s"` | Maximum backoff between HTTP retries | +| `max_open_retries` | `u32` | `5` | Total attempts for index-exists check during `open()` | +| `open_retry_max_delay` | `String` | `"30s"` | Maximum backoff during `open()` probes | +| `circuit_breaker_threshold` | `u32` | `5` | Consecutive poll failures before circuit opens | +| `circuit_breaker_cool_down` | `String` | `"60s"` | Duration to skip polls when circuit is open | + +See [docs/RESILIENCE.md](docs/RESILIENCE.md) for retry, circuit breaker, at-least-once, and backfill semantics. ### File-backed state (optional) @@ -85,7 +94,7 @@ Each call to `poll()`: 1. Issues `POST /{index}/_search` with the query below. 2. Maps each hit's `_source` to a JSON `ProducedMessage`. -3. Updates the `search_after` cursor to the sort tuple of the last successfully published hit. +3. Updates the `search_after` cursor to the sort tuple of the last hit with a valid sort tuple. 4. Returns `ProducedMessages` containing the messages and a serialized `ConnectorState`. 5. Sleeps `polling_interval` before returning. @@ -111,12 +120,11 @@ Two sort keys give stable order when timestamps collide. `_id` is the tiebreaker For each hit in the response: -1. **Missing sort tuple** — skip with `warn!`; cursor not advanced. -2. **Missing `_source`** — skip with `warn!`; cursor not advanced. -3. Both present — serialize `_source` as JSON payload; advance cursor to this hit's sort tuple. +1. **Missing sort tuple** — skip with `warn!` for that hit; if no hit in the batch has a valid sort tuple, the poll fails with `Error::Storage`. +2. **Missing `_source`** — skip with `warn!` (not published); cursor still advances to that hit's sort position. +3. **Both present** — serialize `_source` as JSON payload. -The cursor (`search_after`) only advances for hits where **both** sort and `_source` are present. -An empty batch leaves the cursor unchanged. +The cursor (`search_after`) advances for any hit with a valid sort tuple, including hits skipped for missing `_source`. An empty batch leaves the cursor unchanged. ### Timestamp parsing @@ -225,8 +233,10 @@ Requirements for correct operation: parallel export. Offset paging (`from`/`size`) is not used. - **At-least-once delivery** — no deduplication by `_id`. The in-memory cursor advances before the runtime persists `ConnectorState`; a crash can re-emit the last batch. -- **No HTTP retry** — transient OpenSearch errors fail the poll immediately. No circuit - breaker (unlike the InfluxDB source). + See [docs/RESILIENCE.md](docs/RESILIENCE.md). +- **HTTP retry and circuit breaker** — transient `429`/`5xx` and network errors are retried + per `max_retries`; consecutive failures trip a circuit breaker that skips polls until + cool-down. Permanent errors (`4xx` except `429`) are not retried. - **Backfill gap** — documents indexed with `timestamp_field` values older than the current cursor are not read until connector state is reset. - **Full `_source` only** — entire document JSON is published; no field @@ -239,11 +249,8 @@ Requirements for correct operation: advances past them. If `_source` later becomes available for a document at the same `(timestamp_field, _id)` sort position, it will not be re-fetched. Ensure `_source` is enabled in the index mapping before using this connector. -- **Missing sort tuple causes no cursor advance** — hits returned without a sort tuple - (rare; typically deleted-doc artifacts or partial shard results) are skipped with a - `warn!`. If such hits appear at the tail of a batch, the cursor stays at the last - successfully published document. Subsequent polls return the same hits until OpenSearch - stops including them. +- **Missing sort tuple** — individual hits without a sort tuple are skipped. A batch where + **no** hit has a valid sort tuple fails the poll with `Error::Storage`. - **Single-node transport** — `SingleNodeConnectionPool` to `url`; no cluster node sniffing. diff --git a/core/connectors/sources/opensearch_source/docs/RESILIENCE.md b/core/connectors/sources/opensearch_source/docs/RESILIENCE.md new file mode 100644 index 0000000000..cad7a3a9da --- /dev/null +++ b/core/connectors/sources/opensearch_source/docs/RESILIENCE.md @@ -0,0 +1,176 @@ +# OpenSearch Source — Resilience & Delivery Semantics + +Design reference for HTTP retry, circuit breaker, at-least-once delivery, and +backfill behavior. Phase 1 (retry + circuit breaker) is implemented in the +connector; later phases are documented for future work. + +## Contents + +- [HTTP retry and circuit breaker](#http-retry-and-circuit-breaker) +- [At-least-once delivery](#at-least-once-delivery) +- [Backfill gap](#backfill-gap) +- [Implementation phases](#implementation-phases) +- [Non-goals](#non-goals) + +## HTTP retry and circuit breaker + +### Problem + +Each poll performs HTTP calls to OpenSearch (`HEAD` index exists at `open()`, +`POST /{index}/_search` at `poll()`). Without retry, transient `503`, `429`, or +network blips fail the poll immediately. The connector only retries on the next +`polling_interval` cycle. + +### Constraint + +InfluxDB uses `reqwest` + `build_retry_client` middleware. This connector uses +`opensearch-rs` (`TransportBuilder`). Retry is implemented at the **plugin level** +using `iggy_connector_sdk::retry` helpers (`exponential_backoff`, `jitter`, +`parse_retry_after`). + +### Configuration (`plugin_config`) + +| Field | Default | Purpose | +| ----- | ------- | ------- | +| `max_retries` | `3` | Total attempts per HTTP operation during `poll()` | +| `retry_delay` | `1s` | Base backoff between attempts | +| `retry_max_delay` | `30s` | Cap per retry wait | +| `max_open_retries` | `5` | Startup attempts for index-exists check in `open()` | +| `open_retry_max_delay` | `30s` | Cap for open probe backoff | +| `circuit_breaker_threshold` | `5` | Consecutive failures before circuit opens | +| `circuit_breaker_cool_down` | `60s` | Circuit open duration | + +All fields are optional (`#[serde(default)]`); omitted keys use the defaults above. + +### Retry policy + +**Transient (retry):** network errors; HTTP `429`; HTTP `5xx`; `Retry-After` on `429` +when parseable. + +**Permanent (no retry):** `400`, `401`, `403`, `404`; malformed responses; +search DSL errors. + +### Circuit breaker + +When open, `poll()` skips the search, logs a warning, sleeps `polling_interval`, +and returns an empty batch with `state: None` so the runtime does not persist a +new cursor (same pattern as InfluxDB source). + +Consecutive poll failures (after retries exhausted) increment the breaker. +Successful search resets it. + +## At-least-once delivery + +### Contract + +The connector provides **at-least-once** delivery toward Iggy. Duplicates can +occur when: + +- The process crashes after `finalize_poll` advances the in-memory cursor but + before the runtime persists `ConnectorState`. +- The connector restarts from a cursor behind the last published batch. + +The runtime saves `ConnectorState` only after a successful Iggy send. + +### Mitigations (future phases) + +| Tier | Action | Status | +| ---- | ------ | ------ | +| A | Document contract + recommend consumer dedup on `_id` | Partial (README) | +| B | Set `ProducedMessage.id` from OpenSearch `_id` | Planned | +| C | Runtime ack before cursor advance | Deferred (FFI change) | + +### Consumer guidance + +- Dedup on OpenSearch `_id` plus index name, or a business key in `_source`. +- Treat replays as normal for log pipelines; use upsert sinks where needed. + +## Backfill gap + +### Contract + +Pagination is forward-only on `(timestamp_field asc, _id asc)` via `search_after`. +After catch-up, documents indexed with `timestamp_field` **older** than the +current cursor are not read until connector state is reset. + +### Operator modes + +| Mode | Use case | +| ---- | -------- | +| Forward tail (default) | Live streaming; timestamps track ingest order | +| Bounded backfill | One-time load via time-window `query` | +| Full rescan | Reset runtime `ConnectorState` (duplicates possible) | + +### Future phases + +| Tier | Feature | +| ---- | ------- | +| B | `backfill.mode = warn` — log when index min timestamp < cursor | +| C | `backfill.mode = window` — explicit time-window scan | +| E | `min_timestamp` on fresh start only | + +## Implementation phases + +| Phase | Scope | Status | +| ----- | ----- | ------ | +| 1 | HTTP retry + circuit breaker | **Implemented** | +| 2 | At-least-once + backfill runbook in README | Partial | +| 3 | `ProducedMessage.id` from `_id`; backfill warn mode | Planned | +| 4 | Backfill window mode | Planned | +| 5 | Exactly-once / runtime ack | Deferred | + +## Testing retry and circuit breaker + +Two layers cover resilience behavior. Use both: plugin tests are fast and precise; +integration tests exercise the full connectors runtime + FFI poll loop. + +### Plugin HTTP tests (fast, no Docker) + +In-process axum mock server in `src/http_tests.rs`. Run: + +```bash +cargo test -p iggy_connector_opensearch_source given_transient_search_errors_when_poll_should_retry_and_succeed +cargo test -p iggy_connector_opensearch_source given_circuit_breaker_open_when_poll_should_return_empty_without_error +``` + +These tests inject `503`/`500` responses and assert retry counts and circuit-breaker +skip behavior at the plugin `poll()` boundary. + +### Integration tests (runtime + wiremock) + +Fixtures in `core/integration/tests/connectors/fixtures/opensearch/resilience.rs` +start a [wiremock](https://github.com/LukeMathWalker/wiremock-rs) `MockServer` (no +OpenSearch container) and point the connector URL at it via runtime env overrides: + +| Fixture | Behavior | +| ------- | -------- | +| `OpenSearchSourceTransientErrorFixture` | Two `503` search responses, then one hit | +| `OpenSearchSourceCircuitBreakerFixture` | Persistent `500` with fast retry settings | + +Circuit-breaker **open/skip** semantics (empty poll, no cursor advance) are asserted in +plugin `http_tests.rs` because they are precise to time and poll count. The integration +fixture verifies the runtime keeps the source `Running`, performs HTTP retries, and does +not publish messages under sustained failures. + +Run: + +```bash +cargo test -p integration -- connectors::opensearch::opensearch_source_resilience +``` + +Tests live in `opensearch_source_resilience.rs` and verify message production after +retry, plus runtime `Running` status with no messages under sustained failures. + +### Manual / staging checks + +Point `url` at a proxy or fault-injection layer in front of a real cluster. Tune +`max_retries`, `retry_delay`, `circuit_breaker_threshold`, and `circuit_breaker_cool_down` +in `plugin_config`. Watch connector logs for `OpenSearch request failed; retrying` and +`circuit breaker is OPEN`. + +## Non-goals + +- Exactly-once delivery in the connector alone +- Parallel slice / PIT export (separate design) +- Automatic full-index rescan without operator intent +- Replacing `opensearch-rs` with raw `reqwest` unless plugin retry proves insufficient diff --git a/core/connectors/sources/opensearch_source/src/http_tests.rs b/core/connectors/sources/opensearch_source/src/http_tests.rs index 5f564a4e3c..473ec3cc11 100644 --- a/core/connectors/sources/opensearch_source/src/http_tests.rs +++ b/core/connectors/sources/opensearch_source/src/http_tests.rs @@ -53,6 +53,13 @@ fn base_config(url: &str) -> OpenSearchSourceConfig { timestamp_field: Some("timestamp".to_string()), verbose_logging: false, state: None, + max_retries: Some(1), + retry_delay: Some("1ms".to_string()), + retry_max_delay: Some("10ms".to_string()), + max_open_retries: Some(1), + open_retry_max_delay: Some("10ms".to_string()), + circuit_breaker_threshold: None, + circuit_breaker_cool_down: None, } } @@ -309,7 +316,10 @@ async fn given_batch_where_all_hits_lack_sort_when_poll_should_return_error() { let mut source = OpenSearchSource::new(1, base_config(&base), None); source.open().await.unwrap(); - let error = source.poll().await.expect_err("all-no-sort batch must fail"); + let error = source + .poll() + .await + .expect_err("all-no-sort batch must fail"); assert!( matches!(error, Error::Storage(_)), "expected Storage error, got {error:?}" @@ -412,7 +422,11 @@ async fn given_trailing_hit_without_source_when_poll_should_advance_cursor_past_ source.open().await.unwrap(); let produced = source.poll().await.expect("first poll"); - assert_eq!(produced.messages.len(), 1, "only doc-1 published; doc-2 has no _source"); + assert_eq!( + produced.messages.len(), + 1, + "only doc-1 published; doc-2 has no _source" + ); let cursor = source.test_search_after().await; assert!( @@ -574,3 +588,58 @@ async fn given_verbose_logging_when_poll_should_succeed() { source.open().await.unwrap(); source.poll().await.expect("verbose poll should succeed"); } + +#[tokio::test] +async fn given_transient_search_errors_when_poll_should_retry_and_succeed() { + let request_count = Arc::new(AtomicUsize::new(0)); + let count = request_count.clone(); + + let app = mock_router(StatusCode::OK, move |_| { + let count = count.clone(); + async move { + let attempt = count.fetch_add(1, Ordering::SeqCst); + if attempt < 2 { + (StatusCode::SERVICE_UNAVAILABLE, "temporary".to_string()) + } else { + (StatusCode::OK, search_response(vec![]).to_string()) + } + } + }); + let base = start_server(app).await; + let mut config = base_config(&base); + config.max_retries = Some(3); + config.retry_delay = Some("1ms".to_string()); + let mut source = OpenSearchSource::new(1, config, None); + source.open().await.unwrap(); + + source + .poll() + .await + .expect("poll should succeed after retries"); + assert!( + request_count.load(Ordering::SeqCst) >= 3, + "search should be retried after transient 503" + ); +} + +#[tokio::test] +async fn given_circuit_breaker_open_when_poll_should_return_empty_without_error() { + let app = mock_router(StatusCode::OK, |_| async move { + (StatusCode::INTERNAL_SERVER_ERROR, "boom".to_string()) + }); + let base = start_server(app).await; + let mut config = base_config(&base); + config.circuit_breaker_threshold = Some(1); + config.circuit_breaker_cool_down = Some("60s".to_string()); + let mut source = OpenSearchSource::new(1, config, None); + source.open().await.unwrap(); + + let _ = source.poll().await.expect_err("first poll should fail"); + + let produced = source + .poll() + .await + .expect("open circuit should skip search"); + assert!(produced.messages.is_empty()); + assert!(produced.state.is_none()); +} diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs index 9ebd937720..6857404a9b 100644 --- a/core/connectors/sources/opensearch_source/src/lib.rs +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -18,7 +18,7 @@ */ use async_trait::async_trait; use iggy_common::{DateTime, Utc}; -use iggy_connector_sdk::retry::parse_duration; +use iggy_connector_sdk::retry::{CircuitBreaker, parse_duration}; use iggy_connector_sdk::{ ConnectorState, Error, ProducedMessage, ProducedMessages, Schema, Source, source_connector, }; @@ -30,11 +30,18 @@ use opensearch::{ use secrecy::{ExposeSecret, SecretString}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; +use std::sync::Arc; use std::time::Duration; use tokio::{sync::Mutex, time::sleep}; use tracing::{debug, error, info, warn}; +mod retry; mod state_manager; +use crate::retry::{ + DEFAULT_CB_COOL_DOWN, DEFAULT_CB_THRESHOLD, DEFAULT_MAX_OPEN_RETRIES, DEFAULT_MAX_RETRIES, + DEFAULT_OPEN_RETRY_MAX_DELAY, DEFAULT_RETRY_DELAY, DEFAULT_RETRY_MAX_DELAY, RetryBackoff, + is_transient_status, normalized_max_attempts, sleep_before_retry, +}; use crate::state_manager::{SOURCE_STATE_VERSION, SourceState, validate_state_storage_config}; source_connector!(OpenSearchSource); @@ -101,6 +108,13 @@ pub struct OpenSearchSourceConfig { #[serde(default)] pub verbose_logging: bool, pub state: Option, + pub max_retries: Option, + pub retry_delay: Option, + pub retry_max_delay: Option, + pub max_open_retries: Option, + pub open_retry_max_delay: Option, + pub circuit_breaker_threshold: Option, + pub circuit_breaker_cool_down: Option, } #[derive(Debug)] @@ -111,6 +125,12 @@ pub struct OpenSearchSource { polling_interval: Duration, search_query: Value, verbose: bool, + max_retries: u32, + retry_delay: Duration, + retry_max_delay: Duration, + max_open_retries: u32, + open_retry_max_delay: Duration, + circuit_breaker: Arc, state: Mutex, /// `Some(cause)` when runtime state restore was rejected; `None` means restore succeeded. state_restore_error: Option, @@ -137,9 +157,35 @@ impl OpenSearchSource { let (restored_state, state_restore_error, runtime_state_restored) = restore_state(id, state); + let cb_threshold = config + .circuit_breaker_threshold + .unwrap_or(DEFAULT_CB_THRESHOLD); + let cb_cool_down = parse_duration( + config.circuit_breaker_cool_down.as_deref(), + DEFAULT_CB_COOL_DOWN, + ); + let circuit_breaker = Arc::new(CircuitBreaker::new(cb_threshold, cb_cool_down)); + let max_retries = + normalized_max_attempts(config.max_retries.unwrap_or(DEFAULT_MAX_RETRIES)); + let retry_delay = parse_duration(config.retry_delay.as_deref(), DEFAULT_RETRY_DELAY); + let retry_max_delay = + parse_duration(config.retry_max_delay.as_deref(), DEFAULT_RETRY_MAX_DELAY); + let max_open_retries = + normalized_max_attempts(config.max_open_retries.unwrap_or(DEFAULT_MAX_OPEN_RETRIES)); + let open_retry_max_delay = parse_duration( + config.open_retry_max_delay.as_deref(), + DEFAULT_OPEN_RETRY_MAX_DELAY, + ); + OpenSearchSource { id, config, + max_retries, + retry_delay, + retry_max_delay, + max_open_retries, + open_retry_max_delay, + circuit_breaker, client: None, polling_interval, search_query, @@ -234,6 +280,166 @@ impl OpenSearchSource { Ok(OpenSearch::new(transport)) } + async fn check_index_exists_with_retry(&self, client: &OpenSearch) -> Result<(), Error> { + let max_attempts = self.max_open_retries; + let mut attempt = 0u32; + + loop { + attempt += 1; + let response = match client + .indices() + .exists(opensearch::indices::IndicesExistsParts::Index(&[&self + .config + .index])) + .send() + .await + { + Ok(response) => response, + Err(error) => { + if attempt < max_attempts { + sleep_before_retry( + "index_exists", + self.id, + attempt, + max_attempts, + &RetryBackoff { + delay: self.retry_delay, + max_delay: self.open_retry_max_delay, + }, + None, + &error.to_string(), + ) + .await; + continue; + } + return Err(Error::Storage(format!( + "Failed to check index existence: {error}" + ))); + } + }; + + if response.status_code().is_success() { + return Ok(()); + } + + let status = response.status_code().as_u16(); + if status == 404 { + return Err(Error::InitError(format!( + "Index '{}' does not exist or is not accessible", + self.config.index + ))); + } + + let retry_after = response + .headers() + .get("retry-after") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let error_text = response + .text() + .await + .unwrap_or_else(|_| "unknown error".to_string()); + + if is_transient_status(status) && attempt < max_attempts { + sleep_before_retry( + "index_exists", + self.id, + attempt, + max_attempts, + &RetryBackoff { + delay: self.retry_delay, + max_delay: self.open_retry_max_delay, + }, + retry_after.as_deref(), + &format!("HTTP {status}: {error_text}"), + ) + .await; + continue; + } + + return Err(Error::InitError(format!( + "Index '{}' does not exist or is not accessible", + self.config.index + ))); + } + } + + async fn send_search_with_retry( + &self, + client: &OpenSearch, + search_body: Value, + ) -> Result { + let max_attempts = self.max_retries; + let mut attempt = 0u32; + + loop { + attempt += 1; + let response = match client + .search(SearchParts::Index(&[&self.config.index])) + .body(search_body.clone()) + .send() + .await + { + Ok(response) => response, + Err(error) => { + if attempt < max_attempts { + sleep_before_retry( + "search", + self.id, + attempt, + max_attempts, + &RetryBackoff { + delay: self.retry_delay, + max_delay: self.retry_max_delay, + }, + None, + &error.to_string(), + ) + .await; + continue; + } + return Err(Error::Storage(format!("Failed to execute search: {error}"))); + } + }; + + if response.status_code().is_success() { + return Ok(response); + } + + let status = response.status_code().as_u16(); + let retry_after = response + .headers() + .get("retry-after") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let error_text = response + .text() + .await + .unwrap_or_else(|_| "unknown error".to_string()); + + if is_transient_status(status) && attempt < max_attempts { + sleep_before_retry( + "search", + self.id, + attempt, + max_attempts, + &RetryBackoff { + delay: self.retry_delay, + max_delay: self.retry_max_delay, + }, + retry_after.as_deref(), + &format!("HTTP {status}: {error_text}"), + ) + .await; + continue; + } + + return Err(Error::Storage(format!( + "Search request failed: {error_text}" + ))); + } + } + async fn search_documents(&self, client: &OpenSearch) -> Result { let state = self.state.lock().await; let batch_size = self.batch_size(); @@ -254,22 +460,7 @@ impl OpenSearchSource { search_body["search_after"] = json!(cursor); } - let response = client - .search(SearchParts::Index(&[&self.config.index])) - .body(search_body) - .send() - .await - .map_err(|e| Error::Storage(format!("Failed to execute search: {e}")))?; - - if !response.status_code().is_success() { - let error_text = response - .text() - .await - .map_err(|e| Error::Storage(format!("Failed to read search error body: {e}")))?; - return Err(Error::Storage(format!( - "Search request failed: {error_text}" - ))); - } + let response = self.send_search_with_retry(client, search_body).await?; let mut response_body: Value = response .json() @@ -289,7 +480,11 @@ impl OpenSearchSource { let mut last_poll_timestamp = None; for hit in &hits { - let Some(sort) = hit.get("sort").and_then(|s| s.as_array()).filter(|a| !a.is_empty()) else { + let Some(sort) = hit + .get("sort") + .and_then(|s| s.as_array()) + .filter(|a| !a.is_empty()) + else { warn!( connector_id = self.id, hit_id = hit.get("_id").and_then(|value| value.as_str()), @@ -423,6 +618,20 @@ impl OpenSearchSource { async fn test_last_poll_timestamp(&self) -> Option> { self.state.lock().await.last_poll_timestamp } + + async fn handle_poll_error(&self, error: Error) -> Result { + self.circuit_breaker.record_failure().await; + let mut state = self.state.lock().await; + state.error_count += 1; + state.last_error = Some(error.to_string()); + drop(state); + error!( + "{CONNECTOR_NAME} connector ID: {} poll failed: {error}", + self.id + ); + sleep(self.polling_interval).await; + Err(error) + } } fn restore_state(id: u32, state: Option) -> (State, Option, bool) { @@ -505,21 +714,7 @@ impl Source for OpenSearchSource { let client = self.create_client().await?; - let response = client - .indices() - .exists(opensearch::indices::IndicesExistsParts::Index(&[&self - .config - .index])) - .send() - .await - .map_err(|e| Error::Storage(format!("Failed to check index existence: {e}")))?; - - if !response.status_code().is_success() { - return Err(Error::InitError(format!( - "Index '{}' does not exist or is not accessible", - self.config.index - ))); - } + self.check_index_exists_with_retry(&client).await?; self.client = Some(client); @@ -551,6 +746,19 @@ impl Source for OpenSearchSource { } async fn poll(&self) -> Result { + if self.circuit_breaker.is_open().await { + warn!( + "{CONNECTOR_NAME} connector ID: {} — circuit breaker is OPEN. Skipping poll.", + self.id + ); + sleep(self.polling_interval).await; + return Ok(ProducedMessages { + schema: Schema::Json, + messages: vec![], + state: None, + }); + } + let start_time = std::time::Instant::now(); let client = self @@ -558,28 +766,21 @@ impl Source for OpenSearchSource { .as_ref() .ok_or_else(|| Error::Storage("OpenSearch client not initialized".to_string()))?; - let (messages, persisted_state) = match self.search_documents(client).await { + match self.search_documents(client).await { Ok(outcome) => { + self.circuit_breaker.record_success(); let processing_time = start_time.elapsed().as_millis() as f64; - self.finalize_poll(outcome, processing_time).await - } - Err(e) => { - let mut state = self.state.lock().await; - state.error_count += 1; - state.last_error = Some(e.to_string()); - drop(state); + let (messages, persisted_state) = + self.finalize_poll(outcome, processing_time).await; sleep(self.polling_interval).await; - return Err(e); + Ok(ProducedMessages { + schema: Schema::Json, + messages, + state: persisted_state, + }) } - }; - - sleep(self.polling_interval).await; - - Ok(ProducedMessages { - schema: Schema::Json, - messages, - state: persisted_state, - }) + Err(error) => self.handle_poll_error(error).await, + } } async fn close(&mut self) -> Result<(), Error> { @@ -629,6 +830,13 @@ mod tests { timestamp_field: Some("timestamp".to_string()), verbose_logging: false, state: None, + max_retries: None, + retry_delay: None, + retry_max_delay: None, + max_open_retries: None, + open_retry_max_delay: None, + circuit_breaker_threshold: None, + circuit_breaker_cool_down: None, } } diff --git a/core/connectors/sources/opensearch_source/src/retry.rs b/core/connectors/sources/opensearch_source/src/retry.rs new file mode 100644 index 0000000000..9e947ae5f8 --- /dev/null +++ b/core/connectors/sources/opensearch_source/src/retry.rs @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use iggy_connector_sdk::retry::{exponential_backoff, jitter, parse_retry_after}; +use std::time::Duration; +use tokio::time::sleep; +use tracing::warn; + +pub(crate) const DEFAULT_MAX_RETRIES: u32 = 3; +pub(crate) const DEFAULT_RETRY_DELAY: &str = "1s"; +pub(crate) const DEFAULT_RETRY_MAX_DELAY: &str = "30s"; +pub(crate) const DEFAULT_MAX_OPEN_RETRIES: u32 = 5; +pub(crate) const DEFAULT_OPEN_RETRY_MAX_DELAY: &str = "30s"; +pub(crate) const DEFAULT_CB_THRESHOLD: u32 = 5; +pub(crate) const DEFAULT_CB_COOL_DOWN: &str = "60s"; + +/// Total attempt count (minimum 1), consistent with InfluxDB connector config. +pub(crate) fn normalized_max_attempts(max_retries: u32) -> u32 { + max_retries.max(1) +} + +/// Returns true for HTTP status codes worth retrying: 429 and 5xx. +pub(crate) fn is_transient_status(status: u16) -> bool { + status == 429 || (500..600).contains(&status) +} + +pub(crate) struct RetryBackoff { + pub delay: Duration, + pub max_delay: Duration, +} + +pub(crate) async fn sleep_before_retry( + operation: &str, + connector_id: u32, + attempt: u32, + max_attempts: u32, + backoff: &RetryBackoff, + retry_after: Option<&str>, + reason: &str, +) { + let delay = retry_after.and_then(parse_retry_after).unwrap_or_else(|| { + jitter(exponential_backoff( + backoff.delay, + attempt, + backoff.max_delay, + )) + }); + warn!( + connector_id, + operation, + attempt, + max_attempts, + delay_ms = delay.as_millis(), + reason, + "OpenSearch request failed; retrying" + ); + sleep(delay).await; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn given_429_or_5xx_when_check_transient_should_return_true() { + assert!(is_transient_status(429)); + assert!(is_transient_status(503)); + assert!(!is_transient_status(404)); + assert!(!is_transient_status(400)); + } + + #[test] + fn given_zero_max_retries_when_normalize_should_return_one() { + assert_eq!(normalized_max_attempts(0), 1); + assert_eq!(normalized_max_attempts(3), 3); + } +} diff --git a/core/integration/tests/connectors/fixtures/mod.rs b/core/integration/tests/connectors/fixtures/mod.rs index 34a7da9245..63e77ecb8d 100644 --- a/core/integration/tests/connectors/fixtures/mod.rs +++ b/core/integration/tests/connectors/fixtures/mod.rs @@ -70,8 +70,9 @@ pub use mongodb::{ MongoDbSinkFixture, MongoDbSinkJsonFixture, MongoDbSinkWriteConcernFixture, }; pub use opensearch::{ - OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture, - OpenSearchSourceSmallBatchFixture, OpenSearchSourceTypedFieldsFixture, + OpenSearchSourceCircuitBreakerFixture, OpenSearchSourceMissingIndexFixture, + OpenSearchSourcePreCreatedFixture, OpenSearchSourceSmallBatchFixture, + OpenSearchSourceTransientErrorFixture, OpenSearchSourceTypedFieldsFixture, }; pub use postgres::{ PostgresOps, PostgresSinkByteaFixture, PostgresSinkFixture, PostgresSinkJsonFixture, diff --git a/core/integration/tests/connectors/fixtures/opensearch/container.rs b/core/integration/tests/connectors/fixtures/opensearch/container.rs index b8aa480889..81440b7aa8 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/container.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/container.rs @@ -49,6 +49,20 @@ pub const ENV_SOURCE_STREAMS_0_STREAM: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH pub const ENV_SOURCE_STREAMS_0_TOPIC: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_STREAMS_0_TOPIC"; pub const ENV_SOURCE_STREAMS_0_SCHEMA: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_STREAMS_0_SCHEMA"; pub const ENV_SOURCE_PATH: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PATH"; +pub const ENV_SOURCE_MAX_RETRIES: &str = + "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_MAX_RETRIES"; +pub const ENV_SOURCE_RETRY_DELAY: &str = + "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_RETRY_DELAY"; +pub const ENV_SOURCE_RETRY_MAX_DELAY: &str = + "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_RETRY_MAX_DELAY"; +pub const ENV_SOURCE_MAX_OPEN_RETRIES: &str = + "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_MAX_OPEN_RETRIES"; +pub const ENV_SOURCE_OPEN_RETRY_MAX_DELAY: &str = + "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_OPEN_RETRY_MAX_DELAY"; +pub const ENV_SOURCE_CIRCUIT_BREAKER_THRESHOLD: &str = + "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_CIRCUIT_BREAKER_THRESHOLD"; +pub const ENV_SOURCE_CIRCUIT_BREAKER_COOL_DOWN: &str = + "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_CIRCUIT_BREAKER_COOL_DOWN"; #[allow(dead_code)] #[derive(Debug, Deserialize)] diff --git a/core/integration/tests/connectors/fixtures/opensearch/mod.rs b/core/integration/tests/connectors/fixtures/opensearch/mod.rs index d8a0296ef2..d690e8a4cf 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/mod.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/mod.rs @@ -18,8 +18,12 @@ */ pub mod container; +pub mod resilience; pub mod source; +pub use resilience::{ + OpenSearchSourceCircuitBreakerFixture, OpenSearchSourceTransientErrorFixture, +}; pub use source::{ OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture, OpenSearchSourceSmallBatchFixture, OpenSearchSourceTypedFieldsFixture, diff --git a/core/integration/tests/connectors/fixtures/opensearch/resilience.rs b/core/integration/tests/connectors/fixtures/opensearch/resilience.rs new file mode 100644 index 0000000000..a107737111 --- /dev/null +++ b/core/integration/tests/connectors/fixtures/opensearch/resilience.rs @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use super::container::{ + DEFAULT_TEST_STREAM, DEFAULT_TEST_TOPIC, ENV_SOURCE_BATCH_SIZE, ENV_SOURCE_INDEX, + ENV_SOURCE_MAX_RETRIES, ENV_SOURCE_OPEN_RETRY_MAX_DELAY, ENV_SOURCE_PATH, + ENV_SOURCE_POLLING_INTERVAL, ENV_SOURCE_RETRY_DELAY, ENV_SOURCE_RETRY_MAX_DELAY, + ENV_SOURCE_STREAMS_0_SCHEMA, ENV_SOURCE_STREAMS_0_STREAM, ENV_SOURCE_STREAMS_0_TOPIC, + ENV_SOURCE_TIMESTAMP_FIELD, ENV_SOURCE_URL, +}; +use async_trait::async_trait; +use integration::harness::{TestBinaryError, TestFixture}; +use std::collections::HashMap; +use wiremock::matchers::{method, path_regex}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +pub const RESILIENCE_INDEX: &str = "resilience_test"; + +fn index_path_pattern() -> &'static str { + r"/resilience_test$" +} + +fn search_path_pattern() -> &'static str { + r"/resilience_test/_search" +} + +fn search_success_body() -> serde_json::Value { + serde_json::json!({ + "hits": { + "hits": [{ + "_id": "doc-retry-1", + "_source": { + "id": 42, + "name": "retry_doc", + "value": 100, + "timestamp": "2024-01-15T12:00:00.000Z" + }, + "sort": ["2024-01-15T12:00:00.000Z", "doc-retry-1"] + }] + } + }) +} + +fn resilience_base_envs(mock_uri: &str) -> HashMap { + let mut envs = HashMap::new(); + envs.insert(ENV_SOURCE_URL.to_string(), mock_uri.to_string()); + envs.insert(ENV_SOURCE_INDEX.to_string(), RESILIENCE_INDEX.to_string()); + envs.insert(ENV_SOURCE_POLLING_INTERVAL.to_string(), "100ms".to_string()); + envs.insert(ENV_SOURCE_BATCH_SIZE.to_string(), "10".to_string()); + envs.insert( + ENV_SOURCE_TIMESTAMP_FIELD.to_string(), + "timestamp".to_string(), + ); + envs.insert( + ENV_SOURCE_STREAMS_0_STREAM.to_string(), + DEFAULT_TEST_STREAM.to_string(), + ); + envs.insert( + ENV_SOURCE_STREAMS_0_TOPIC.to_string(), + DEFAULT_TEST_TOPIC.to_string(), + ); + envs.insert(ENV_SOURCE_STREAMS_0_SCHEMA.to_string(), "json".to_string()); + envs.insert( + ENV_SOURCE_PATH.to_string(), + "../../target/debug/libiggy_connector_opensearch_source".to_string(), + ); + envs.insert(ENV_SOURCE_RETRY_DELAY.to_string(), "50ms".to_string()); + envs.insert(ENV_SOURCE_RETRY_MAX_DELAY.to_string(), "200ms".to_string()); + envs.insert( + ENV_SOURCE_OPEN_RETRY_MAX_DELAY.to_string(), + "200ms".to_string(), + ); + envs +} + +async fn mount_index_exists(mock_server: &MockServer) { + Mock::given(method("HEAD")) + .and(path_regex(index_path_pattern())) + .respond_with(ResponseTemplate::new(200)) + .mount(mock_server) + .await; +} + +async fn mount_transient_search_mocks(mock_server: &MockServer) { + Mock::given(method("POST")) + .and(path_regex(search_path_pattern())) + .respond_with(ResponseTemplate::new(503).set_body_string("temporary")) + .up_to_n_times(2) + .with_priority(1) + .mount(mock_server) + .await; + + Mock::given(method("POST")) + .and(path_regex(search_path_pattern())) + .respond_with(ResponseTemplate::new(200).set_body_json(search_success_body())) + .with_priority(2) + .mount(mock_server) + .await; +} + +async fn mount_persistent_search_failure(mock_server: &MockServer) { + Mock::given(method("POST")) + .and(path_regex(search_path_pattern())) + .respond_with(ResponseTemplate::new(500).set_body_string("persistent")) + .mount(mock_server) + .await; +} + +/// Wiremock-backed fixture: two transient `503` search responses, then one hit. +pub struct OpenSearchSourceTransientErrorFixture { + mock_server: MockServer, +} + +impl OpenSearchSourceTransientErrorFixture { + pub async fn search_request_count(&self) -> usize { + self.mock_server + .received_requests() + .await + .unwrap_or_default() + .iter() + .filter(|request| { + request.url.path().contains(search_path_pattern()) + && request.method.as_str() == "POST" + }) + .count() + } +} + +#[async_trait] +impl TestFixture for OpenSearchSourceTransientErrorFixture { + async fn setup() -> Result { + let mock_server = MockServer::start().await; + mount_index_exists(&mock_server).await; + mount_transient_search_mocks(&mock_server).await; + + Ok(Self { mock_server }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + let mut envs = resilience_base_envs(&self.mock_server.uri()); + envs.insert(ENV_SOURCE_MAX_RETRIES.to_string(), "5".to_string()); + envs + } +} + +/// Wiremock-backed fixture: every search returns `500` with a low circuit-breaker threshold. +pub struct OpenSearchSourceCircuitBreakerFixture { + mock_server: MockServer, +} + +impl OpenSearchSourceCircuitBreakerFixture { + pub async fn search_request_count(&self) -> usize { + self.mock_server + .received_requests() + .await + .unwrap_or_default() + .iter() + .filter(|request| { + request.url.path().contains(search_path_pattern()) + && request.method.as_str() == "POST" + }) + .count() + } +} + +#[async_trait] +impl TestFixture for OpenSearchSourceCircuitBreakerFixture { + async fn setup() -> Result { + let mock_server = MockServer::start().await; + mount_index_exists(&mock_server).await; + mount_persistent_search_failure(&mock_server).await; + + Ok(Self { mock_server }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + resilience_base_envs(&self.mock_server.uri()) + } +} diff --git a/core/integration/tests/connectors/opensearch/mod.rs b/core/integration/tests/connectors/opensearch/mod.rs index abf01cea52..656ab91645 100644 --- a/core/integration/tests/connectors/opensearch/mod.rs +++ b/core/integration/tests/connectors/opensearch/mod.rs @@ -18,6 +18,7 @@ */ mod opensearch_source; +mod opensearch_source_resilience; mod opensearch_source_types; const TEST_MESSAGE_COUNT: usize = 3; diff --git a/core/integration/tests/connectors/opensearch/opensearch_source_resilience.rs b/core/integration/tests/connectors/opensearch/opensearch_source_resilience.rs new file mode 100644 index 0000000000..c03f67d8d5 --- /dev/null +++ b/core/integration/tests/connectors/opensearch/opensearch_source_resilience.rs @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use super::{POLL_ATTEMPTS, POLL_INTERVAL_MS}; +use crate::connectors::fixtures::{ + OpenSearchSourceCircuitBreakerFixture, OpenSearchSourceTransientErrorFixture, +}; +use iggy_common::MessageClient; +use iggy_common::{Consumer, Identifier, PollingStrategy}; +use iggy_connector_sdk::api::ConnectorStatus; +use integration::harness::seeds; +use integration::iggy_harness; +use reqwest::Client; +use std::time::Duration; +use tokio::time::sleep; + +async fn poll_json_messages( + client: &impl MessageClient, + consumer_id: &str, + min_messages: usize, +) -> Vec { + let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); + let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); + let consumer: Identifier = consumer_id.try_into().unwrap(); + let mut received = Vec::new(); + + for _ in 0..POLL_ATTEMPTS { + if let Ok(polled) = client + .poll_messages( + &stream_id, + &topic_id, + None, + &Consumer::new(consumer.clone()), + &PollingStrategy::next(), + 10, + true, + ) + .await + { + received.clear(); + for msg in polled.messages { + if let Ok(json) = serde_json::from_slice(&msg.payload) { + received.push(json); + } + } + if received.len() >= min_messages { + break; + } + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + + received +} + +async fn wait_for_source_status( + http_client: &Client, + api_address: &str, + expected: ConnectorStatus, +) -> ConnectorStatus { + let mut status = ConnectorStatus::Starting; + for _ in 0..POLL_ATTEMPTS { + let response = http_client + .get(format!("{api_address}/sources")) + .send() + .await + .expect("Failed to query /sources"); + assert_eq!(response.status(), 200); + let sources: Vec = + response.json().await.expect("Failed to parse sources"); + if let Some(source) = sources.first() { + status = source.status; + if status == expected { + break; + } + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + status +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/opensearch/source_resilience.toml")), + seed = seeds::connector_stream +)] +async fn given_transient_search_errors_when_connector_polls_should_retry_and_produce( + harness: &TestHarness, + fixture: OpenSearchSourceTransientErrorFixture, +) { + let client = harness.root_client().await.unwrap(); + let received = poll_json_messages(&client, "resilience_retry_consumer", 1).await; + + assert_eq!( + received.len(), + 1, + "expected one message after transient search errors were retried" + ); + assert_eq!( + received[0].get("id").and_then(|value| value.as_i64()), + Some(42) + ); + assert!( + fixture.search_request_count().await >= 3, + "search should be retried after transient 503 responses" + ); + + let api_address = harness + .connectors_runtime() + .expect("connector runtime should be available") + .http_url(); + let status = + wait_for_source_status(&Client::new(), &api_address, ConnectorStatus::Running).await; + assert_eq!(status, ConnectorStatus::Running); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/opensearch/source_resilience.toml")), + seed = seeds::connector_stream +)] +async fn given_persistent_search_errors_when_connector_polls_should_remain_running_without_messages( + harness: &TestHarness, + fixture: OpenSearchSourceCircuitBreakerFixture, +) { + let client = harness.root_client().await.unwrap(); + let api_address = harness + .connectors_runtime() + .expect("connector runtime should be available") + .http_url(); + let http_client = Client::new(); + + sleep(Duration::from_millis(POLL_INTERVAL_MS * 20)).await; + + let status = wait_for_source_status(&http_client, &api_address, ConnectorStatus::Running).await; + assert_eq!( + status, + ConnectorStatus::Running, + "persistent search failures should not move source to Error" + ); + + let received = poll_json_messages(&client, "resilience_cb_consumer", 1).await; + assert!( + received.is_empty(), + "persistent failures should not produce messages, got {}", + received.len() + ); + + assert!( + fixture.search_request_count().await >= 3, + "connector should retry transient HTTP failures before surfacing poll errors" + ); +} diff --git a/core/integration/tests/connectors/opensearch/plugin_config/config.toml b/core/integration/tests/connectors/opensearch/plugin_config/config.toml new file mode 100644 index 0000000000..d463accac6 --- /dev/null +++ b/core/integration/tests/connectors/opensearch/plugin_config/config.toml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "source" +key = "opensearch" +enabled = true +version = 0 +name = "OpenSearch source (resilience)" +path = "../../target/debug/libiggy_connector_opensearch_source" +plugin_config_format = "json" + +[[streams]] +stream = "test_stream" +topic = "test_topic" +schema = "json" +batch_length = 1000 +linger_time = "5ms" + +[plugin_config] +url = "http://localhost:9200" +index = "resilience_test" +polling_interval = "100ms" +batch_size = 10 +timestamp_field = "timestamp" +max_retries = 3 +retry_delay = "50ms" +retry_max_delay = "200ms" +max_open_retries = 3 +open_retry_max_delay = "200ms" +circuit_breaker_threshold = 1 +circuit_breaker_cool_down = "5s" diff --git a/core/integration/tests/connectors/opensearch/source_resilience.toml b/core/integration/tests/connectors/opensearch/source_resilience.toml new file mode 100644 index 0000000000..8487d4a0f4 --- /dev/null +++ b/core/integration/tests/connectors/opensearch/source_resilience.toml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[connectors] +config_type = "local" +config_dir = "tests/connectors/opensearch/plugin_config" From cedc4a38655e8f3265530d03d82deeac083456b6 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Fri, 19 Jun 2026 12:49:02 -0400 Subject: [PATCH 13/19] opensearch: precompute search body, improve errors Refactor OpenSearch source initialization and polling: add max_retries/max_open_retries config options and pre-build a search_body_base during open() instead of rebuilding the query each poll. Harden error handling by returning Connection errors for uninitialized client, converting some Init/Storage errors, and refusing to advance the cursor when an entire batch has valid sort tuples but every hit lacks _source (to avoid losing data). Other changes: optimize send_search_with_retry to avoid unnecessary clones on the final attempt; treat numeric timestamps >1e12 as milliseconds (keep seconds otherwise); capture/clone a state snapshot before persisting and adjust avg batch time calculation to use saturating subtraction; increment error_count/last_error on poll failures; make file state writes race-safe by adding PID to temp file suffix and use sync_all; add serde default for state version. Update tests and README formatting, and remove the humantime dependency from the Opensearch source crate. --- Cargo.lock | 1 - .../sources/opensearch_source/Cargo.toml | 1 - .../sources/opensearch_source/README.md | 2 +- .../opensearch_source/src/http_tests.rs | 26 +++-- .../sources/opensearch_source/src/lib.rs | 105 ++++++++++++------ .../opensearch_source/src/state_manager.rs | 6 +- .../opensearch_source_resilience.rs | 1 - 7 files changed, 94 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c134f2b6e..adc597a4a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6939,7 +6939,6 @@ dependencies = [ "async-trait", "axum", "dashmap", - "humantime", "iggy_common", "iggy_connector_sdk", "once_cell", diff --git a/core/connectors/sources/opensearch_source/Cargo.toml b/core/connectors/sources/opensearch_source/Cargo.toml index b0fb38eb1b..361065dbd2 100644 --- a/core/connectors/sources/opensearch_source/Cargo.toml +++ b/core/connectors/sources/opensearch_source/Cargo.toml @@ -42,7 +42,6 @@ crate-type = ["cdylib", "lib"] [dependencies] async-trait = { workspace = true } dashmap = { workspace = true } -humantime = { workspace = true } iggy_common = { workspace = true } iggy_connector_sdk = { workspace = true } once_cell = { workspace = true } diff --git a/core/connectors/sources/opensearch_source/README.md b/core/connectors/sources/opensearch_source/README.md index 2b27f4a1c0..ea3786af05 100644 --- a/core/connectors/sources/opensearch_source/README.md +++ b/core/connectors/sources/opensearch_source/README.md @@ -41,7 +41,7 @@ index = "logs-*" polling_interval = "10s" batch_size = 100 timestamp_field = "@timestamp" -query = { "match_all": {} } +query = { match_all = {} } ``` ### Required fields diff --git a/core/connectors/sources/opensearch_source/src/http_tests.rs b/core/connectors/sources/opensearch_source/src/http_tests.rs index 473ec3cc11..901ca73153 100644 --- a/core/connectors/sources/opensearch_source/src/http_tests.rs +++ b/core/connectors/sources/opensearch_source/src/http_tests.rs @@ -371,7 +371,7 @@ async fn given_cursor_set_when_empty_poll_should_preserve_cursor() { } #[tokio::test] -async fn given_hit_without_source_when_search_should_skip_document() { +async fn given_all_hits_missing_source_when_poll_should_return_error() { let hit = json!({ "_id": "doc-1", "sort": ["2024-01-01T00:00:00Z", "doc-1"] @@ -385,11 +385,18 @@ async fn given_hit_without_source_when_search_should_skip_document() { let mut source = OpenSearchSource::new(1, base_config(&base), None); source.open().await.unwrap(); - let produced = source.poll().await.expect("poll should succeed"); - assert!(produced.messages.is_empty()); + // All hits have valid sort tuples but none have _source; cursor must not advance. + let error = source + .poll() + .await + .expect_err("all-_source-absent batch must error"); assert!( - source.test_search_after().await.is_some(), - "_source-missing skip must still advance cursor to that hit's sort position" + matches!(error, Error::Storage(_)), + "expected Storage error, got {error:?}" + ); + assert!( + source.test_search_after().await.is_none(), + "cursor must not advance when entire batch has no _source" ); } @@ -449,10 +456,10 @@ async fn given_trailing_hit_without_source_when_poll_should_advance_cursor_past_ } #[tokio::test] -async fn given_poll_without_open_should_return_storage_error() { +async fn given_poll_without_open_should_return_connection_error() { let source = OpenSearchSource::new(1, base_config("http://127.0.0.1:9"), None); let error = source.poll().await.expect_err("poll without open"); - assert!(matches!(error, Error::Storage(_))); + assert!(matches!(error, Error::Connection(_))); } #[tokio::test] @@ -558,8 +565,11 @@ async fn given_runtime_state_when_open_should_not_load_stale_file_state() { #[tokio::test] async fn given_epoch_seconds_timestamp_should_update_last_poll_timestamp() { - let hit = search_hit("doc-1", "1705312200", json!({})); + let hit = search_hit("doc-1", "2024-01-15T10:00:00Z", json!({})); let mut hit = hit; + // Override _source.timestamp with epoch-seconds integer to exercise the numeric + // branch of parse_document_timestamp. Sort tuple stays as the RFC3339 string that + // search_hit() placed there — sort is used only for cursor positioning. hit["_source"]["timestamp"] = json!(1_705_312_200_i64); let body = search_response(vec![hit]).to_string(); let app = mock_router(StatusCode::OK, move |_| { diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs index 6857404a9b..927c96a2eb 100644 --- a/core/connectors/sources/opensearch_source/src/lib.rs +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -108,9 +108,11 @@ pub struct OpenSearchSourceConfig { #[serde(default)] pub verbose_logging: bool, pub state: Option, + /// Total poll-phase attempt count (not retry count); 1 = no retries, 3 = 2 retries. pub max_retries: Option, pub retry_delay: Option, pub retry_max_delay: Option, + /// Total open-phase attempt count (not retry count); 1 = no retries, 5 = 4 retries. pub max_open_retries: Option, pub open_retry_max_delay: Option, pub circuit_breaker_threshold: Option, @@ -123,7 +125,9 @@ pub struct OpenSearchSource { config: OpenSearchSourceConfig, client: Option, polling_interval: Duration, - search_query: Value, + /// Pre-built query body (query + size + sort). Set once in `open()` after config validation. + /// `None` before `open()` and after a failed `open()`. + search_body_base: Option, verbose: bool, max_retries: u32, retry_delay: Duration, @@ -149,10 +153,6 @@ impl OpenSearchSource { pub fn new(id: u32, config: OpenSearchSourceConfig, state: Option) -> Self { let polling_interval = parse_duration(config.polling_interval.as_deref(), DEFAULT_POLLING_INTERVAL); - let search_query = config - .query - .clone() - .unwrap_or_else(|| json!({ "match_all": {} })); let verbose = config.verbose_logging; let (restored_state, state_restore_error, runtime_state_restored) = restore_state(id, state); @@ -188,7 +188,7 @@ impl OpenSearchSource { circuit_breaker, client: None, polling_interval, - search_query, + search_body_base: None, verbose, state: Mutex::new(restored_state), state_restore_error, @@ -302,6 +302,9 @@ impl OpenSearchSource { self.id, attempt, max_attempts, + // retry_delay is shared with the poll phase (no separate open-phase + // base delay); open_retry_max_delay caps the growth independently. + // Same pattern as the InfluxDB source connector. &RetryBackoff { delay: self.retry_delay, max_delay: self.open_retry_max_delay, @@ -312,7 +315,7 @@ impl OpenSearchSource { .await; continue; } - return Err(Error::Storage(format!( + return Err(Error::InitError(format!( "Failed to check index existence: {error}" ))); } @@ -358,7 +361,7 @@ impl OpenSearchSource { } return Err(Error::InitError(format!( - "Index '{}' does not exist or is not accessible", + "Index '{}' check failed: HTTP {status}: {error_text}", self.config.index ))); } @@ -367,16 +370,21 @@ impl OpenSearchSource { async fn send_search_with_retry( &self, client: &OpenSearch, - search_body: Value, + mut search_body: Value, ) -> Result { let max_attempts = self.max_retries; let mut attempt = 0u32; loop { attempt += 1; + let body = if attempt < max_attempts { + search_body.clone() + } else { + std::mem::take(&mut search_body) + }; let response = match client .search(SearchParts::Index(&[&self.config.index])) - .body(search_body.clone()) + .body(body) .send() .await { @@ -442,19 +450,16 @@ impl OpenSearchSource { async fn search_documents(&self, client: &OpenSearch) -> Result { let state = self.state.lock().await; - let batch_size = self.batch_size(); - let timestamp_field = self.timestamp_field(); let search_after = state.search_after.clone(); drop(state); - let mut search_body = json!({ - "query": self.search_query, - "size": batch_size, - "sort": [ - { timestamp_field: { "order": "asc" } }, - { "_id": { "order": "asc" } } - ] - }); + let timestamp_field = self.timestamp_field(); + + let mut search_body = self + .search_body_base + .as_ref() + .ok_or_else(|| Error::Connection("connector not initialized; call open() first".to_string()))? + .clone(); if let Some(cursor) = search_after { search_body["search_after"] = json!(cursor); @@ -476,7 +481,7 @@ impl OpenSearchSource { let mut messages = Vec::with_capacity(hits.len()); let mut batch_bytes = 0u64; - let mut last_sort: Option<&Vec> = None; + let mut last_sort = None; let mut last_poll_timestamp = None; for hit in &hits { @@ -532,6 +537,17 @@ impl OpenSearchSource { ))); } + // Guard: cursor would advance past the entire batch but nothing would be published. + // This happens when _source is disabled on the index (all sort-bearing hits lack _source). + if !hits.is_empty() && messages.is_empty() && last_sort.is_some() { + return Err(Error::Storage(format!( + "OpenSearch returned {} hit(s) with valid sort tuples but all were missing \ + _source; index may have _source disabled. Refusing to advance cursor \ + without publishing any messages.", + hits.len() + ))); + } + Ok(SearchOutcome { messages, search_after: last_sort.map(ToOwned::to_owned), @@ -565,16 +581,20 @@ impl OpenSearchSource { let total_polls = state.processing_stats.successful_polls_count + state.processing_stats.empty_polls_count; + // total_polls >= 1 because one of the two counters was just incremented above, + // but use saturating_sub to make the invariant machine-checked. state.processing_stats.avg_batch_processing_time_ms = - (state.processing_stats.avg_batch_processing_time_ms * (total_polls - 1) as f64 + (state.processing_stats.avg_batch_processing_time_ms + * total_polls.saturating_sub(1) as f64 + processing_time_ms) / total_polls as f64; let produced_count = outcome.messages.len(); let total_documents_published = state.total_documents_published; + let state_snapshot = state.clone(); let messages = outcome.messages; - let persisted_state = self.serialize_state(&state); drop(state); + let persisted_state = self.serialize_state(&state_snapshot); if self.verbose { info!( @@ -622,6 +642,9 @@ impl OpenSearchSource { async fn handle_poll_error(&self, error: Error) -> Result { self.circuit_breaker.record_failure().await; let mut state = self.state.lock().await; + // error_count and last_error accumulate in State and are captured by the next + // finalize_poll call (success path). Both runtime ConnectorState and file state + // preserve these values across restarts. state.error_count += 1; state.last_error = Some(error.to_string()); drop(state); @@ -687,7 +710,10 @@ fn parse_document_timestamp(value: &Value) -> Option> { .map(|timestamp| timestamp.with_timezone(&Utc)), Value::Number(number) => { let raw = number.as_i64()?; - let millis = if raw.abs() > 1_000_000_000_000 { + // Values above 1e12 are already milliseconds (Unix epoch seconds won't reach + // 1e12 until year 33658). Values at or below are treated as seconds and + // multiplied by 1000. + let millis = if raw > 1_000_000_000_000 || raw < -1_000_000_000_000 { raw } else { raw.saturating_mul(1_000) @@ -707,6 +733,17 @@ impl Source for OpenSearchSource { validate_open_config(&self.config)?; + let timestamp_field = self.timestamp_field(); + let batch_size = self.batch_size(); + self.search_body_base = Some(json!({ + "query": self.config.query.clone().unwrap_or_else(|| json!({ "match_all": {} })), + "size": batch_size, + "sort": [ + { timestamp_field: { "order": "asc" } }, + { "_id": { "order": "asc" } } + ] + })); + info!( "Opening OpenSearch source connector with ID: {} for URL: {}, index: {}", self.id, self.config.url, self.config.index @@ -764,7 +801,7 @@ impl Source for OpenSearchSource { let client = self .client .as_ref() - .ok_or_else(|| Error::Storage("OpenSearch client not initialized".to_string()))?; + .ok_or_else(|| Error::Connection("OpenSearch client not initialized".to_string()))?; match self.search_documents(client).await { Ok(outcome) => { @@ -859,7 +896,7 @@ mod tests { } #[test] - fn given_persisted_state_should_restore_total_documents_published() { + fn given_persisted_runtime_state_when_new_should_restore_counts() { let state = test_state(); let serialized = rmp_serde::to_vec(&state).expect("Failed to serialize state"); let connector_state = ConnectorState(serialized); @@ -881,7 +918,7 @@ mod tests { } #[test] - fn given_no_state_should_start_fresh() { + fn given_no_runtime_state_when_new_should_start_fresh() { let source = OpenSearchSource::new(1, test_config(), None); let runtime = tokio::runtime::Runtime::new().unwrap(); @@ -896,7 +933,7 @@ mod tests { } #[test] - fn given_invalid_state_should_set_state_restore_error() { + fn given_invalid_runtime_state_when_new_should_set_restore_error() { let invalid_state = ConnectorState(b"not valid msgpack".to_vec()); let source = OpenSearchSource::new(1, test_config(), Some(invalid_state)); @@ -938,7 +975,7 @@ mod tests { } #[test] - fn given_unparsable_timestamp_value_should_return_none() { + fn given_unparsable_timestamp_when_parsed_should_return_none() { let value = json!("not-a-timestamp"); assert!(parse_document_timestamp(&value).is_none()); } @@ -988,7 +1025,7 @@ mod tests { } #[test] - fn given_unsupported_file_state_version_should_fail() { + fn given_unsupported_file_state_version_when_applied_should_fail() { use crate::state_manager::SourceState; let runtime = tokio::runtime::Runtime::new().unwrap(); @@ -1053,19 +1090,19 @@ mod tests { } #[test] - fn given_rfc3339_timestamp_value_should_parse() { + fn given_rfc3339_timestamp_when_parsed_should_succeed() { let value = json!("2024-01-15T10:30:00Z"); assert!(parse_document_timestamp(&value).is_some()); } #[test] - fn given_epoch_millis_timestamp_value_should_parse() { + fn given_epoch_millis_timestamp_when_parsed_should_succeed() { let value = json!(1_705_312_200_000_i64); assert!(parse_document_timestamp(&value).is_some()); } #[test] - fn given_state_should_round_trip_serialization() { + fn given_state_when_serialized_should_round_trip() { let original = test_state(); let serialized = rmp_serde::to_vec(&original).expect("Failed to serialize"); @@ -1082,7 +1119,7 @@ mod tests { } #[test] - fn given_state_when_serialize_helper_should_produce_connector_state() { + fn given_state_when_serialized_should_produce_connector_state() { let source = OpenSearchSource::new(1, test_config(), None); let state = test_state(); diff --git a/core/connectors/sources/opensearch_source/src/state_manager.rs b/core/connectors/sources/opensearch_source/src/state_manager.rs index 994735e6cf..5e980246c9 100644 --- a/core/connectors/sources/opensearch_source/src/state_manager.rs +++ b/core/connectors/sources/opensearch_source/src/state_manager.rs @@ -105,6 +105,7 @@ pub(crate) fn create_state_storage(config: &StateConfig) -> Result, + #[serde(default)] pub version: u32, pub data: serde_json::Value, pub metadata: Option, @@ -144,7 +145,8 @@ impl StateStorage for FileStateStorage { .map_err(|e| Error::Storage(format!("Failed to create state directory: {e}")))?; let path = self.get_state_path(&state.id); - let tmp_path = path.with_extension("json.tmp"); + // PID in the suffix avoids races when two processes write the same state_id. + let tmp_path = path.with_extension(format!("json.{}.tmp", std::process::id())); let json = serde_json::to_string(state) .map_err(|e| Error::Serialization(format!("Failed to serialize source state: {e}")))?; @@ -161,7 +163,7 @@ impl StateStorage for FileStateStorage { "Failed to write state temp file: {e}" ))); } - if let Err(e) = tmp_file.sync_data().await { + if let Err(e) = tmp_file.sync_all().await { let _ = fs::remove_file(&tmp_path).await; return Err(Error::Storage(format!( "Failed to sync state temp file: {e}" diff --git a/core/integration/tests/connectors/opensearch/opensearch_source_resilience.rs b/core/integration/tests/connectors/opensearch/opensearch_source_resilience.rs index c03f67d8d5..c4577b55eb 100644 --- a/core/integration/tests/connectors/opensearch/opensearch_source_resilience.rs +++ b/core/integration/tests/connectors/opensearch/opensearch_source_resilience.rs @@ -53,7 +53,6 @@ async fn poll_json_messages( ) .await { - received.clear(); for msg in polled.messages { if let Ok(json) = serde_json::from_slice(&msg.payload) { received.push(json); From 0d6c2d8fe0e276c2e9741178fa9e61834e7f234d Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Fri, 19 Jun 2026 21:08:39 -0400 Subject: [PATCH 14/19] chore(ci): sync .github files from apache/iggy master Bring CI actions, workflows, and hawkeye.version in line with upstream. --- .github/actions/php/pre-merge/action.yml | 58 ++++++- .../python-maturin/pre-merge/action.yml | 1 + .github/actions/rust/pre-merge/action.yml | 42 +++-- .../actions/utils/free-disk-space/action.yml | 21 ++- .../utils/setup-rust-with-cache/action.yml | 36 +++- .github/config/components.yml | 16 ++ .github/config/hawkeye.version | 1 + .github/dependabot.yml | 105 +++++++----- .github/workflows/_build_python_wheels.yml | 8 +- .github/workflows/_build_rust_artifacts.yml | 6 +- .github/workflows/_common.yml | 160 ++++++++++-------- .github/workflows/_detect.yml | 2 +- .github/workflows/_publish_rust_crates.yml | 2 +- .github/workflows/_test.yml | 28 ++- .github/workflows/_test_bdd.yml | 2 +- .github/workflows/_test_examples.yml | 4 +- .github/workflows/coverage-baseline.yml | 64 ++++--- .github/workflows/edge-release.yml | 2 +- .github/workflows/issue-labeler-assigner.yml | 2 +- .github/workflows/post-merge.yml | 6 +- .github/workflows/publish.yml | 18 +- 21 files changed, 396 insertions(+), 188 deletions(-) create mode 100644 .github/config/hawkeye.version diff --git a/.github/actions/php/pre-merge/action.yml b/.github/actions/php/pre-merge/action.yml index 698dd3ec22..9f4b117cea 100644 --- a/.github/actions/php/pre-merge/action.yml +++ b/.github/actions/php/pre-merge/action.yml @@ -65,6 +65,12 @@ runs: shell: bash run: echo "CARGO_TARGET_DIR=${GITHUB_WORKSPACE}/target" >> "$GITHUB_ENV" + - name: Install cargo-llvm-cov + if: inputs.task == 'test' + uses: taiki-e/install-action@v2 + with: + tool: cargo-llvm-cov + - name: Validate task shell: bash run: | @@ -120,7 +126,14 @@ runs: if: inputs.task == 'test' shell: bash run: | - cargo build --manifest-path foreign/php/Cargo.toml + cd foreign/php + source <(cargo llvm-cov show-env --sh) + export CARGO_TARGET_DIR=$CARGO_LLVM_COV_TARGET_DIR + + echo "LLVM_PROFILE_FILE=${LLVM_PROFILE_FILE}" >> "$GITHUB_ENV" + + cargo llvm-cov clean --workspace + cargo build --manifest-path Cargo.toml extension="$(find "${CARGO_TARGET_DIR}/debug" -maxdepth 1 -name 'libiggy_php.so' -print -quit)" if [ -z "$extension" ]; then echo "PHP extension was not produced" @@ -150,7 +163,9 @@ runs: IGGY_PORT: 8090 IGGY_USERNAME: iggy IGGY_PASSWORD: iggy - run: ./scripts/test.sh + run: | + mkdir -p ../../reports + ./scripts/test.sh --log-junit ../../reports/php-junit.xml - name: Stop Iggy server (plain) if: always() && inputs.task == 'test' @@ -180,7 +195,32 @@ runs: IGGY_PORT: 8090 IGGY_TLS_CONNECTION_STRING: iggy+tcp://iggy:iggy@127.0.0.1:8090?tls=true&tls_domain=localhost&tls_ca_file=${{ github.workspace }}/core/certs/iggy_ca_cert.pem IGGY_TLS_PLAINTEXT_ADDRESS: 127.0.0.1:8090 - run: ./scripts/test.sh tests/TlsTest.php + run: ./scripts/test.sh --log-junit ../../reports/php-tls-junit.xml tests/TlsTest.php + + - name: Generate PHP coverage report + if: always() && inputs.task == 'test' + shell: bash + run: | + cd foreign/php + source <(cargo llvm-cov show-env --sh) + export CARGO_TARGET_DIR=$CARGO_LLVM_COV_TARGET_DIR + + cargo llvm-cov report --lcov \ + --ignore-filename-regex='(\.cargo/|/rustc/|/core/)' \ + --output-path ../../reports/php-coverage.lcov + + repo_root="$(git rev-parse --show-toplevel)" + + # Fix paths: cargo-llvm-cov can output paths relative to the crate root (src/...) + # or absolute repo paths (.../foreign/php/src/...), but Codecov expects repo-root + # paths (foreign/php/src/...). + sed -i \ + -e "s|^SF:${repo_root}/foreign/php/src/|SF:foreign/php/src/|" \ + -e 's|^SF:src/|SF:foreign/php/src/|' \ + ../../reports/php-coverage.lcov + + echo "Coverage report generated: $(wc -l < ../../reports/php-coverage.lcov) lines" + ../../scripts/ci/validate-lcov.sh ../../reports/php-coverage.lcov - name: Stop Iggy server (TLS) if: always() && inputs.task == 'test' @@ -188,3 +228,15 @@ runs: with: pid-file: ${{ steps.iggy-tls.outputs.pid_file }} log-file: ${{ steps.iggy-tls.outputs.log_file }} + + - name: Upload test artifacts + if: always() && inputs.task == 'test' + uses: actions/upload-artifact@v7 + with: + name: php-test-results-${{ github.run_id }}-${{ github.run_attempt }} + path: | + reports/php-junit.xml + reports/php-tls-junit.xml + reports/php-coverage.lcov + retention-days: 7 + if-no-files-found: ignore diff --git a/.github/actions/python-maturin/pre-merge/action.yml b/.github/actions/python-maturin/pre-merge/action.yml index 0eacd05879..25409e5ac8 100644 --- a/.github/actions/python-maturin/pre-merge/action.yml +++ b/.github/actions/python-maturin/pre-merge/action.yml @@ -182,6 +182,7 @@ runs: sed -i 's|^SF:src/|SF:foreign/python/src/|' ../../reports/python-coverage.lcov echo "Coverage report generated: $(wc -l < ../../reports/python-coverage.lcov) lines" + ../../scripts/ci/validate-lcov.sh ../../reports/python-coverage.lcov shell: bash - name: Upload test artifacts diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index c7388b9093..43175bbb0c 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -37,23 +37,35 @@ runs: # subtree; isolate its cache from the stable `dev` namespace so the # two don't evict each other. shared-key: ${{ inputs.task == 'miri' && 'miri' || 'dev' }} - print-cache-status: ${{ startsWith(inputs.task, 'test-') }} + # fmt/sort/machete never build into target/, so the multi-GB cache + # restore is pure overhead. Compiling legs (check/clippy/doctest/test-*) + # keep it; a cold cache there recompiles the whole dep tree. + read-cache: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'machete') && 'false' || 'true' }} + # Light legs fit in the runner's ~89 GiB default headroom; skip the + # ~20-45s reclaim. Disk-heavy legs (coverage build + testcontainers + # images on test-*, cross-builds, miri, verify-publish) keep it. + free-disk-space: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'clippy' || inputs.task == 'check' || inputs.task == 'machete' || inputs.task == 'doctest') && 'false' || 'true' }} + + - name: Install cargo-sort + if: inputs.task == 'sort' + uses: taiki-e/install-action@v2 + with: + tool: cargo-sort + - name: Install cargo-machete + if: inputs.task == 'machete' + uses: taiki-e/install-action@v2 + with: + tool: cargo-machete + + # cargo-http-registry has no prebuilt artifact in taiki-e/install-action, + # so it stays a source build. verify-publish is a low-frequency task. - name: Install tools for specific tasks + if: inputs.task == 'verify-publish' run: | - case "${{ inputs.task }}" in - sort) - cargo install cargo-sort --locked - ;; - machete) - cargo install cargo-machete --locked - ;; - verify-publish) - if ! command -v cargo-http-registry >/dev/null 2>&1; then - cargo install cargo-http-registry --locked - fi - ;; - esac + if ! command -v cargo-http-registry >/dev/null 2>&1; then + cargo install cargo-http-registry --locked + fi shell: bash # DAG-based test scoping: use cargo-rail to compute affected crates from the @@ -157,7 +169,7 @@ runs: - name: Install dependencies for Rust tests if: startsWith(inputs.task, 'test-') && runner.os == 'Linux' run: | - sudo apt-get update --yes && sudo apt-get install --yes musl-tools gnome-keyring keyutils dbus-x11 libsecret-tools + sudo apt-get install --yes musl-tools gnome-keyring keyutils dbus-x11 libsecret-tools rm -f $HOME/.local/share/keyrings/* shell: bash diff --git a/.github/actions/utils/free-disk-space/action.yml b/.github/actions/utils/free-disk-space/action.yml index b683284fe8..a2e815c1ed 100644 --- a/.github/actions/utils/free-disk-space/action.yml +++ b/.github/actions/utils/free-disk-space/action.yml @@ -21,11 +21,20 @@ description: >- pre-installed toolchains the Iggy build does not use. Logs reclaimed GiB and elapsed seconds. +inputs: + aggressive: + description: >- + When "true", run jlumbroso/free-disk-space for a deep cleanup + (~30+ GiB) instead of the light built-in removal (~12 GiB). + Enable only for disk-heavy jobs (e.g. llvm-cov instrumented builds). + required: false + default: "false" + runs: using: "composite" steps: - - name: Cleanup disk space - if: runner.os == 'Linux' + - name: Cleanup disk space (light) + if: runner.os == 'Linux' && inputs.aggressive != 'true' shell: bash run: | echo "Disk space before cleanup:" @@ -51,3 +60,11 @@ runs: echo "::notice::Reclaimed ${reclaimed_gib} GiB in $((end - start))s" echo "Disk space after cleanup:" df -h + + - name: Cleanup disk space (aggressive) + if: runner.os == 'Linux' && inputs.aggressive == 'true' + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + # Preserve the .NET SDK when actions/setup-dotnet provisioned it + # (DOTNET_ROOT set); removing it breaks subsequent dotnet invocations. + dotnet: ${{ env.DOTNET_ROOT == '' }} diff --git a/.github/actions/utils/setup-rust-with-cache/action.yml b/.github/actions/utils/setup-rust-with-cache/action.yml index 43367dc7c4..e01ce81f8d 100644 --- a/.github/actions/utils/setup-rust-with-cache/action.yml +++ b/.github/actions/utils/setup-rust-with-cache/action.yml @@ -31,8 +31,17 @@ inputs: description: "Whether to save cache (true/false)" required: false default: "false" - print-cache-status: - description: "Whether to print cache status to job summary" + free-disk-space: + description: >- + Whether to reclaim runner disk before the build. Default "true". + Set "false" for light legs (fmt/sort/clippy/check/machete/doctest) that + fit in the ~89 GiB default headroom and should not pay the ~20-45s cost. + required: false + default: "true" + free-disk-space-aggressive: + description: >- + Forwarded to free-disk-space. When "true", run jlumbroso for a deep + cleanup. Enable for disk-heavy jobs (e.g. llvm-cov instrumented builds). required: false default: "false" @@ -40,7 +49,10 @@ runs: using: "composite" steps: - name: Free runner disk space + if: inputs.free-disk-space == 'true' uses: ./.github/actions/utils/free-disk-space + with: + aggressive: ${{ inputs.free-disk-space-aggressive }} - name: Install system dependencies (Linux) if: runner.os == 'Linux' @@ -82,16 +94,26 @@ runs: save-if: ${{ inputs.save-cache == 'true' }} - name: Cache status - if: inputs.read-cache == 'true' && inputs.print-cache-status == 'true' + if: inputs.read-cache == 'true' run: | - echo "### Rust Cache" >> $GITHUB_STEP_SUMMARY + CARGO_HOME_DIR="${CARGO_HOME:-$HOME/.cargo}" if [ "${{ steps.rust-cache.outputs.cache-hit }}" == "true" ]; then - echo "✅ Cache hit" >> $GITHUB_STEP_SUMMARY + status="full hit" elif [ -d "target" ]; then - echo "🔶 Partial cache hit" >> $GITHUB_STEP_SUMMARY + status="partial hit" else - echo "❌ Cache miss" >> $GITHUB_STEP_SUMMARY + status="miss" fi + # Unpacked on-disk footprint of the restore. The cache action only logs + # the compressed size; this prints to the job log (stdout), so size is + # visible without opening the summary and without annotations. + echo "Rust cache: ${status}" + printf '%-10s %s\n' "SIZE" "PATH" + for dir in "${GITHUB_WORKSPACE}/target" "${CARGO_HOME_DIR}/registry" "${CARGO_HOME_DIR}/git"; do + if [ -d "${dir}" ]; then + printf '%-10s %s\n' "$(du -sh "${dir}" 2>/dev/null | cut -f1)" "${dir/#${GITHUB_WORKSPACE}\//}" + fi + done shell: bash - name: Install cargo-nextest diff --git a/.github/config/components.yml b/.github/config/components.yml index fea06a63a6..79b11a255c 100644 --- a/.github/config/components.yml +++ b/.github/config/components.yml @@ -281,6 +281,8 @@ components: paths: - "scripts/run-bdd-tests.sh" - "bdd/docker-compose.yml" + - "bdd/docker-compose.server.yml" + - "bdd/docker-compose.cluster.yml" - "bdd/docker-compose.coverage.yml" # Individual BDD tests per SDK - only run when specific SDK changes @@ -307,6 +309,18 @@ components: - "bdd/scenarios/**" tasks: ["bdd-python"] + bdd-php: + depends_on: + - "rust-server" + - "rust-sdk" + - "sdk-php" + - "bdd-infrastructure" + - "ci-infrastructure" + paths: + - "bdd/php/**" + - "bdd/scenarios/basic_messaging.feature" + tasks: ["bdd-php"] + bdd-go: depends_on: - "rust-server" @@ -446,6 +460,8 @@ components: helm: paths: - "helm/**" + - "scripts/ci/setup-helm-smoke-cluster.sh" + - "scripts/ci/test-helm.sh" tasks: ["validate", "smoke"] rust-bench-dashboard: diff --git a/.github/config/hawkeye.version b/.github/config/hawkeye.version new file mode 100644 index 0000000000..a194c18e86 --- /dev/null +++ b/.github/config/hawkeye.version @@ -0,0 +1 @@ +6.5.1 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 74c872c16b..b61effac1f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,6 +15,10 @@ # specific language governing permissions and limitations # under the License. +# Version updates: every entry uses a single group covering ALL update +# types, so each ecosystem entry opens at most one version-update PR +# per cycle. Security updates bypass these groups and may still open +# individual PRs. version: 2 updates: - package-ecosystem: "cargo" @@ -39,12 +43,9 @@ updates: cooldown: default-days: 7 groups: - minor-and-patch: + rust: patterns: - "*" - update-types: - - "minor" - - "patch" - package-ecosystem: "github-actions" directory: "/" @@ -59,17 +60,14 @@ updates: - "apache/iggy-committers" labels: - "dependencies" - - "github_actions" + - "github-actions" - "CI/CD" cooldown: default-days: 7 groups: - minor-and-patch: + github-actions: patterns: - "*" - update-types: - - "minor" - - "patch" - package-ecosystem: "uv" directories: @@ -92,17 +90,41 @@ updates: cooldown: default-days: 7 groups: - minor-and-patch: + # Load-bearing for Python, not just tidiness: the three projects + # share apache-iggy via a path dependency, so each uv.lock embeds + # the SDK's requirement metadata. Majors escaped the previous + # minor-and-patch group as per-directory PRs that rewrote that + # embedded metadata without the manifest change it was derived + # from, and could never pass the lockfile check (see #3451). + python: patterns: - "*" - update-types: - - "minor" - - "patch" - package-ecosystem: "npm" directories: - - "/foreign/node" - "/web" + schedule: + interval: "weekly" + day: "thursday" + rebase-strategy: "disabled" + open-pull-requests-limit: 3 + commit-message: + prefix: "chore(deps): " + reviewers: + - "apache/iggy-committers" + labels: + - "dependencies" + - "javascript" + cooldown: + default-days: 7 + groups: + web: + patterns: + - "*" + + - package-ecosystem: "npm" + directories: + - "/foreign/node" - "/examples/node" schedule: interval: "weekly" @@ -119,12 +141,9 @@ updates: cooldown: default-days: 7 groups: - minor-and-patch: + node: patterns: - "*" - update-types: - - "minor" - - "patch" - package-ecosystem: "gomod" directories: @@ -146,12 +165,9 @@ updates: cooldown: default-days: 7 groups: - minor-and-patch: + go: patterns: - "*" - update-types: - - "minor" - - "patch" - package-ecosystem: "nuget" directories: @@ -176,12 +192,9 @@ updates: - dependency-name: "Microsoft.SourceLink.GitHub" - dependency-name: "Microsoft.Extensions.Logging.Abstractions" groups: - minor-and-patch: + csharp: patterns: - "*" - update-types: - - "minor" - - "patch" - package-ecosystem: "gradle" directories: @@ -202,13 +215,16 @@ updates: - "java" cooldown: default-days: 7 + ignore: + # examples/java wires iggy's own artifact to the local build via + # dependency substitution (settings.gradle.kts) behind a + # `local-dev` placeholder version. A Dependabot bump to a real + # release breaks example CI, which builds against the local tree. + - dependency-name: "org.apache.iggy:*" groups: - minor-and-patch: + java: patterns: - "*" - update-types: - - "minor" - - "patch" - package-ecosystem: "docker" directories: @@ -243,13 +259,27 @@ updates: - "docker" cooldown: default-days: 7 + ignore: + # Docker dependency-name matching strips the registry prefix, so + # MCR images are named by repository path only. A + # `mcr.microsoft.com/...` form silently never matches and the bump + # leaks through (see closed PRs #3474, #3485). + # + # The Python interpreter version is pinned and synced across SDK + # files, CI, and Dockerfiles by + # scripts/ci/sync-python-interpreter-version.sh; a Dependabot bump + # of the base image alone would fail that check. + - dependency-name: "python" + - dependency-name: "devcontainers/python" + # Go and .NET toolchain base images are pinned deliberately to + # match the versions used across CI and the SDK builds. Bumping + # them is a manual decision, not Dependabot churn. + - dependency-name: "golang" + - dependency-name: "dotnet/sdk" groups: - minor-and-patch: + docker: patterns: - "*" - update-types: - - "minor" - - "patch" - package-ecosystem: "bazel" directory: "/foreign/cpp" @@ -264,13 +294,10 @@ updates: - "apache/iggy-committers" labels: - "dependencies" - - "C++" + - "cpp" cooldown: default-days: 7 groups: - minor-and-patch: + cpp: patterns: - "*" - update-types: - - "minor" - - "patch" diff --git a/.github/workflows/_build_python_wheels.yml b/.github/workflows/_build_python_wheels.yml index c88b337530..d0f2708db8 100644 --- a/.github/workflows/_build_python_wheels.yml +++ b/.github/workflows/_build_python_wheels.yml @@ -72,7 +72,7 @@ jobs: -o /tmp/copy-latest-from-master.sh chmod +x /tmp/copy-latest-from-master.sh - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 with: ref: ${{ inputs.commit }} @@ -134,7 +134,7 @@ jobs: -o /tmp/copy-latest-from-master.sh chmod +x /tmp/copy-latest-from-master.sh - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 with: ref: ${{ inputs.commit }} @@ -195,7 +195,7 @@ jobs: -o /tmp/copy-latest-from-master.sh chmod +x /tmp/copy-latest-from-master.sh - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 with: ref: ${{ inputs.commit }} @@ -238,7 +238,7 @@ jobs: -o /tmp/copy-latest-from-master.sh chmod +x /tmp/copy-latest-from-master.sh - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 with: ref: ${{ inputs.commit }} diff --git a/.github/workflows/_build_rust_artifacts.yml b/.github/workflows/_build_rust_artifacts.yml index 404a501e9c..4902b29b5c 100644 --- a/.github/workflows/_build_rust_artifacts.yml +++ b/.github/workflows/_build_rust_artifacts.yml @@ -88,7 +88,7 @@ jobs: -o /tmp/copy-latest-from-master.sh chmod +x /tmp/copy-latest-from-master.sh - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 with: ref: ${{ inputs.commit || github.sha }} @@ -200,7 +200,7 @@ jobs: -o /tmp/copy-latest-from-master.sh chmod +x /tmp/copy-latest-from-master.sh - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 with: ref: ${{ inputs.commit || github.sha }} @@ -268,7 +268,7 @@ jobs: outputs: artifact_name: ${{ steps.output.outputs.artifact_name }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 with: ref: ${{ inputs.commit || github.sha }} diff --git a/.github/workflows/_common.yml b/.github/workflows/_common.yml index ef6a9c9d84..616008abcb 100644 --- a/.github/workflows/_common.yml +++ b/.github/workflows/_common.yml @@ -28,16 +28,16 @@ jobs: name: Check Rust versions sync runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 - name: Check Rust versions are synchronized - run: ./scripts/ci/sync-rust-version.sh --check + run: ./scripts/ci/sync-rustc-version.sh --check python-versions: name: Check Python SDK versions sync runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 - name: Check Python SDK versions are synchronized run: ./scripts/ci/python-sdk-version-sync.sh --check @@ -46,16 +46,28 @@ jobs: name: Check Python interpreter versions sync runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 - name: Check Python interpreter versions are synchronized - run: ./scripts/ci/sync-python-version.sh --check + run: ./scripts/ci/sync-python-interpreter-version.sh --check + + python-lockfiles: + name: Check Python lockfiles + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + + - name: Check uv.lock files are in sync with pyproject.toml + run: ./scripts/ci/uv-lock-check.sh --check version-consistency: name: Check version consistency runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 - name: Setup yq run: | @@ -74,32 +86,32 @@ jobs: name: Check license headers runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 - - name: Setup Go - uses: ./.github/actions/utils/setup-go-with-cache - with: - go-version: "1.23" - download-deps: "false" + - name: Read HawkEye version + id: hawkeye-version + run: echo "version=$(cat .github/config/hawkeye.version)" >> "$GITHUB_OUTPUT" - - name: Install addlicense - run: go install github.com/google/addlicense@4529cd558fa0bf07ab0f2650e36d089fb1c07c89 + - name: Install HawkEye + uses: taiki-e/install-action@v2.81.8 + with: + tool: hawkeye@${{ steps.hawkeye-version.outputs.version }} - - name: Check Apache license headers + - name: Check license headers run: ./scripts/ci/license-headers.sh --check third-party-licenses: name: Validate third-party licenses runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 - uses: ./.github/actions/utils/validate-third-party-licenses markdown: name: Markdown lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 - name: Setup Node.js uses: actions/setup-node@v6 @@ -117,7 +129,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 - name: Install shellcheck run: | @@ -133,7 +145,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 with: fetch-depth: 0 # Need full history to get diff @@ -145,7 +157,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 with: fetch-depth: 0 # Need full history to get diff @@ -157,7 +169,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 with: fetch-depth: 0 # Need full history to get diff @@ -175,7 +187,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 with: fetch-depth: 0 @@ -190,11 +202,11 @@ jobs: FORCE_COLOR: 1 steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 with: fetch-depth: 0 - name: Check typos - uses: crate-ci/typos@v1.46.2 + uses: crate-ci/typos@v1.47.2 summary: name: Common checks summary @@ -203,6 +215,7 @@ jobs: rust-versions, python-versions, python-interpreter-versions, + python-lockfiles, version-consistency, license-headers, third-party-licenses, @@ -219,34 +232,35 @@ jobs: steps: - name: Summary run: | - echo "## 📋 Common Checks Summary" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "| Check | Status | Description |" >> $GITHUB_STEP_SUMMARY - echo "|-------|--------|-------------|" >> $GITHUB_STEP_SUMMARY + echo "## 📋 Common Checks Summary" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "| Check | Status | Description |" >> "$GITHUB_STEP_SUMMARY" + echo "|-------|--------|-------------|" >> "$GITHUB_STEP_SUMMARY" # Always-run checks RUST_VERSIONS="${{ needs.rust-versions.result }}" PYTHON_VERSIONS="${{ needs.python-versions.result }}" PYTHON_INTERPRETER_VERSIONS="${{ needs.python-interpreter-versions.result }}" + PYTHON_LOCKFILES="${{ needs.python-lockfiles.result }}" VERSION_CONSISTENCY="${{ needs.version-consistency.result }}" LICENSE_HEADERS="${{ needs.license-headers.result }}" THIRD_PARTY_LICENSES="${{ needs.third-party-licenses.result }}" MARKDOWN="${{ needs.markdown.result }}" if [ "$RUST_VERSIONS" = "success" ]; then - echo "| ✅ Rust Versions | success | All Rust versions synchronized |" >> $GITHUB_STEP_SUMMARY + echo "| ✅ Rust Versions | success | All Rust versions synchronized |" >> "$GITHUB_STEP_SUMMARY" elif [ "$RUST_VERSIONS" = "failure" ]; then - echo "| ❌ Rust Versions | failure | Rust versions mismatch in Dockerfiles |" >> $GITHUB_STEP_SUMMARY + echo "| ❌ Rust Versions | failure | Rust versions mismatch in Dockerfiles |" >> "$GITHUB_STEP_SUMMARY" else - echo "| ⏭️ Rust Versions | $RUST_VERSIONS | Check skipped |" >> $GITHUB_STEP_SUMMARY + echo "| ⏭️ Rust Versions | $RUST_VERSIONS | Check skipped |" >> "$GITHUB_STEP_SUMMARY" fi if [ "$PYTHON_VERSIONS" = "success" ]; then - echo "| ✅ Python SDK Versions | success | Cargo.toml and pyproject.toml synchronized |" >> $GITHUB_STEP_SUMMARY + echo "| ✅ Python SDK Versions | success | Cargo.toml and pyproject.toml synchronized |" >> "$GITHUB_STEP_SUMMARY" elif [ "$PYTHON_VERSIONS" = "failure" ]; then - echo "| ❌ Python SDK Versions | failure | Version mismatch between Cargo.toml and pyproject.toml |" >> $GITHUB_STEP_SUMMARY + echo "| ❌ Python SDK Versions | failure | Version mismatch between Cargo.toml and pyproject.toml |" >> "$GITHUB_STEP_SUMMARY" else - echo "| ⏭️ Python SDK Versions | $PYTHON_VERSIONS | Check skipped |" >> $GITHUB_STEP_SUMMARY + echo "| ⏭️ Python SDK Versions | $PYTHON_VERSIONS | Check skipped |" >> "$GITHUB_STEP_SUMMARY" fi if [ "$PYTHON_INTERPRETER_VERSIONS" = "success" ]; then @@ -257,100 +271,108 @@ jobs: echo "| ⏭️ Python Interpreter Versions | $PYTHON_INTERPRETER_VERSIONS | Check skipped |" >> "$GITHUB_STEP_SUMMARY" fi + if [ "$PYTHON_LOCKFILES" = "success" ]; then + echo "| ✅ Python Lockfiles | success | uv.lock files in sync with pyproject.toml |" >> "$GITHUB_STEP_SUMMARY" + elif [ "$PYTHON_LOCKFILES" = "failure" ]; then + echo "| ❌ Python Lockfiles | failure | uv.lock out of sync (run scripts/ci/uv-lock-check.sh --fix) |" >> "$GITHUB_STEP_SUMMARY" + else + echo "| ⏭️ Python Lockfiles | $PYTHON_LOCKFILES | Check skipped |" >> "$GITHUB_STEP_SUMMARY" + fi + if [ "$VERSION_CONSISTENCY" = "success" ]; then - echo "| ✅ Version Consistency | success | Group and workspace dep versions match |" >> $GITHUB_STEP_SUMMARY + echo "| ✅ Version Consistency | success | Group and workspace dep versions match |" >> "$GITHUB_STEP_SUMMARY" elif [ "$VERSION_CONSISTENCY" = "failure" ]; then - echo "| ❌ Version Consistency | failure | Version mismatch detected |" >> $GITHUB_STEP_SUMMARY + echo "| ❌ Version Consistency | failure | Version mismatch detected |" >> "$GITHUB_STEP_SUMMARY" else - echo "| ⏭️ Version Consistency | $VERSION_CONSISTENCY | Check skipped |" >> $GITHUB_STEP_SUMMARY + echo "| ⏭️ Version Consistency | $VERSION_CONSISTENCY | Check skipped |" >> "$GITHUB_STEP_SUMMARY" fi if [ "$LICENSE_HEADERS" = "success" ]; then - echo "| ✅ License Headers | success | All files have Apache headers |" >> $GITHUB_STEP_SUMMARY + echo "| ✅ License Headers | success | All files have Apache headers |" >> "$GITHUB_STEP_SUMMARY" elif [ "$LICENSE_HEADERS" = "failure" ]; then - echo "| ❌ License Headers | failure | Missing Apache license headers |" >> $GITHUB_STEP_SUMMARY + echo "| ❌ License Headers | failure | Missing Apache license headers |" >> "$GITHUB_STEP_SUMMARY" else - echo "| ⏭️ License Headers | $LICENSE_HEADERS | Check skipped |" >> $GITHUB_STEP_SUMMARY + echo "| ⏭️ License Headers | $LICENSE_HEADERS | Check skipped |" >> "$GITHUB_STEP_SUMMARY" fi if [ "$THIRD_PARTY_LICENSES" = "success" ]; then - echo "| ✅ Third-Party Licenses | success | All deps in about.toml accept list |" >> $GITHUB_STEP_SUMMARY + echo "| ✅ Third-Party Licenses | success | All deps in about.toml accept list |" >> "$GITHUB_STEP_SUMMARY" elif [ "$THIRD_PARTY_LICENSES" = "failure" ]; then - echo "| ❌ Third-Party Licenses | failure | Disallowed license detected (see job log) |" >> $GITHUB_STEP_SUMMARY + echo "| ❌ Third-Party Licenses | failure | Disallowed license detected (see job log) |" >> "$GITHUB_STEP_SUMMARY" else - echo "| ⏭️ Third-Party Licenses | $THIRD_PARTY_LICENSES | Check skipped |" >> $GITHUB_STEP_SUMMARY + echo "| ⏭️ Third-Party Licenses | $THIRD_PARTY_LICENSES | Check skipped |" >> "$GITHUB_STEP_SUMMARY" fi if [ "$MARKDOWN" = "success" ]; then - echo "| ✅ Markdown Lint | success | All markdown files are valid |" >> $GITHUB_STEP_SUMMARY + echo "| ✅ Markdown Lint | success | All markdown files are valid |" >> "$GITHUB_STEP_SUMMARY" elif [ "$MARKDOWN" = "failure" ]; then - echo "| ❌ Markdown Lint | failure | Markdown formatting issues found |" >> $GITHUB_STEP_SUMMARY + echo "| ❌ Markdown Lint | failure | Markdown formatting issues found |" >> "$GITHUB_STEP_SUMMARY" else - echo "| ⏭️ Markdown Lint | $MARKDOWN | Check skipped |" >> $GITHUB_STEP_SUMMARY + echo "| ⏭️ Markdown Lint | $MARKDOWN | Check skipped |" >> "$GITHUB_STEP_SUMMARY" fi SHELLCHECK="${{ needs.shellcheck.result }}" if [ "$SHELLCHECK" = "success" ]; then - echo "| ✅ Shellcheck | success | All shell scripts are valid |" >> $GITHUB_STEP_SUMMARY + echo "| ✅ Shellcheck | success | All shell scripts are valid |" >> "$GITHUB_STEP_SUMMARY" elif [ "$SHELLCHECK" = "failure" ]; then - echo "| ❌ Shellcheck | failure | Shell script issues found |" >> $GITHUB_STEP_SUMMARY + echo "| ❌ Shellcheck | failure | Shell script issues found |" >> "$GITHUB_STEP_SUMMARY" else - echo "| ⏭️ Shellcheck | $SHELLCHECK | Check skipped |" >> $GITHUB_STEP_SUMMARY + echo "| ⏭️ Shellcheck | $SHELLCHECK | Check skipped |" >> "$GITHUB_STEP_SUMMARY" fi TRAILING="${{ needs.trailing-whitespace.result }}" if [ "$TRAILING" = "success" ]; then - echo "| ✅ Trailing Whitespace | success | No trailing whitespace found |" >> $GITHUB_STEP_SUMMARY + echo "| ✅ Trailing Whitespace | success | No trailing whitespace found |" >> "$GITHUB_STEP_SUMMARY" elif [ "$TRAILING" = "failure" ]; then - echo "| ❌ Trailing Whitespace | failure | Trailing whitespace detected |" >> $GITHUB_STEP_SUMMARY + echo "| ❌ Trailing Whitespace | failure | Trailing whitespace detected |" >> "$GITHUB_STEP_SUMMARY" else - echo "| ⏭️ Trailing Whitespace | $TRAILING | Check skipped |" >> $GITHUB_STEP_SUMMARY + echo "| ⏭️ Trailing Whitespace | $TRAILING | Check skipped |" >> "$GITHUB_STEP_SUMMARY" fi TRAILING_NL="${{ needs.trailing-newline.result }}" if [ "$TRAILING_NL" = "success" ]; then - echo "| ✅ Trailing Newline | success | All text files have trailing newlines |" >> $GITHUB_STEP_SUMMARY + echo "| ✅ Trailing Newline | success | All text files have trailing newlines |" >> "$GITHUB_STEP_SUMMARY" elif [ "$TRAILING_NL" = "failure" ]; then - echo "| ❌ Trailing Newline | failure | Missing trailing newlines detected |" >> $GITHUB_STEP_SUMMARY + echo "| ❌ Trailing Newline | failure | Missing trailing newlines detected |" >> "$GITHUB_STEP_SUMMARY" else - echo "| ⏭️ Trailing Newline | $TRAILING_NL | Check skipped |" >> $GITHUB_STEP_SUMMARY + echo "| ⏭️ Trailing Newline | $TRAILING_NL | Check skipped |" >> "$GITHUB_STEP_SUMMARY" fi BINARY_ARTIFACTS="${{ needs.binary-artifacts.result }}" if [ "$BINARY_ARTIFACTS" = "success" ]; then - echo "| ✅ Binary Artifacts | success | No binary artifacts found |" >> $GITHUB_STEP_SUMMARY + echo "| ✅ Binary Artifacts | success | No binary artifacts found |" >> "$GITHUB_STEP_SUMMARY" elif [ "$BINARY_ARTIFACTS" = "failure" ]; then - echo "| ❌ Binary Artifacts | failure | Binary artifacts detected in commit |" >> $GITHUB_STEP_SUMMARY + echo "| ❌ Binary Artifacts | failure | Binary artifacts detected in commit |" >> "$GITHUB_STEP_SUMMARY" else - echo "| ⏭️ Binary Artifacts | $BINARY_ARTIFACTS | Check skipped |" >> $GITHUB_STEP_SUMMARY + echo "| ⏭️ Binary Artifacts | $BINARY_ARTIFACTS | Check skipped |" >> "$GITHUB_STEP_SUMMARY" fi TOML_FORMAT="${{ needs.toml-format.result }}" if [ "$TOML_FORMAT" = "success" ]; then - echo "| ✅ TOML Format | success | All TOML files properly formatted |" >> $GITHUB_STEP_SUMMARY + echo "| ✅ TOML Format | success | All TOML files properly formatted |" >> "$GITHUB_STEP_SUMMARY" elif [ "$TOML_FORMAT" = "failure" ]; then - echo "| ❌ TOML Format | failure | TOML formatting issues found |" >> $GITHUB_STEP_SUMMARY + echo "| ❌ TOML Format | failure | TOML formatting issues found |" >> "$GITHUB_STEP_SUMMARY" else - echo "| ⏭️ TOML Format | $TOML_FORMAT | Check skipped |" >> $GITHUB_STEP_SUMMARY + echo "| ⏭️ TOML Format | $TOML_FORMAT | Check skipped |" >> "$GITHUB_STEP_SUMMARY" fi TYPOS="${{ needs.typos.result }}" if [ "$TYPOS" = "success" ]; then - echo "| ✅ Typos Check | success | All files are free of spelling typos |" >> $GITHUB_STEP_SUMMARY + echo "| ✅ Typos Check | success | All files are free of spelling typos |" >> "$GITHUB_STEP_SUMMARY" elif [ "$TYPOS" = "failure" ]; then - echo "| ❌ Typos Check | failure | Spelling typos detected (run 'typos -w' to fix) |" >> $GITHUB_STEP_SUMMARY + echo "| ❌ Typos Check | failure | Spelling typos detected (run 'typos -w' to fix) |" >> "$GITHUB_STEP_SUMMARY" else - echo "| ⏭️ Typos Check | $TYPOS | Check skipped |" >> $GITHUB_STEP_SUMMARY + echo "| ⏭️ Typos Check | $TYPOS | Check skipped |" >> "$GITHUB_STEP_SUMMARY" fi - echo "" >> $GITHUB_STEP_SUMMARY + echo "" >> "$GITHUB_STEP_SUMMARY" # Overall status if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then - echo "### ❌ Some checks failed" >> $GITHUB_STEP_SUMMARY - echo "Please review the failed checks above and fix the issues." >> $GITHUB_STEP_SUMMARY + echo "### ❌ Some checks failed" >> "$GITHUB_STEP_SUMMARY" + echo "Please review the failed checks above and fix the issues." >> "$GITHUB_STEP_SUMMARY" elif [[ "${{ contains(needs.*.result, 'skipped') }}" == "true" ]]; then - echo "### ⚠️ Some checks were skipped" >> $GITHUB_STEP_SUMMARY + echo "### ⚠️ Some checks were skipped" >> "$GITHUB_STEP_SUMMARY" else - echo "### ✅ All checks passed!" >> $GITHUB_STEP_SUMMARY + echo "### ✅ All checks passed!" >> "$GITHUB_STEP_SUMMARY" fi diff --git a/.github/workflows/_detect.yml b/.github/workflows/_detect.yml index 8acde265bd..4e5e44fa42 100644 --- a/.github/workflows/_detect.yml +++ b/.github/workflows/_detect.yml @@ -73,7 +73,7 @@ jobs: examples_matrix: ${{ steps.mk.outputs.examples_matrix }} other_matrix: ${{ steps.mk.outputs.other_matrix }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 with: fetch-depth: 0 diff --git a/.github/workflows/_publish_rust_crates.yml b/.github/workflows/_publish_rust_crates.yml index a33e5a6c2f..ae94da95f6 100644 --- a/.github/workflows/_publish_rust_crates.yml +++ b/.github/workflows/_publish_rust_crates.yml @@ -91,7 +91,7 @@ jobs: echo "✅ Downloaded latest copy script from master" - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 with: # No `|| github.sha` fallback: the `Resolve commit` step below # requires an explicit, non-empty `inputs.commit` so the tag step diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 58c5e8d8b3..7519047f50 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -48,7 +48,7 @@ jobs: timeout-minutes: 60 steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 - name: Skip noop if: inputs.component == 'noop' @@ -64,7 +64,7 @@ jobs: - name: Upload coverage to Codecov if: startsWith(inputs.component, 'rust') && startsWith(inputs.task, 'test-') - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} files: codecov.json @@ -77,7 +77,7 @@ jobs: # Python SDK - name: Set up Docker Buildx for Python if: inputs.component == 'sdk-python' && inputs.task == 'test' - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Run Python SDK task if: inputs.component == 'sdk-python' @@ -87,7 +87,7 @@ jobs: - name: Upload Python coverage to Codecov if: inputs.component == 'sdk-python' && inputs.task == 'test' - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} files: reports/python-coverage.lcov @@ -104,6 +104,18 @@ jobs: with: task: ${{ inputs.task }} + - name: Upload PHP coverage to Codecov + if: inputs.component == 'sdk-php' && inputs.task == 'test' + uses: codecov/codecov-action@v7.0.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: reports/php-coverage.lcov + disable_search: true + flags: php + fail_ci_if_error: false + verbose: true + override_pr: ${{ github.event.pull_request.number }} + # Node SDK - name: Run Node SDK task if: inputs.component == 'sdk-node' @@ -113,7 +125,7 @@ jobs: - name: Upload Node coverage to Codecov if: inputs.component == 'sdk-node' && (inputs.task == 'test' || inputs.task == 'e2e') - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} files: reports/node-coverage/${{ inputs.task == 'test' && 'unit' || 'e2e' }}/lcov.info @@ -132,7 +144,7 @@ jobs: - name: Upload Go coverage to Codecov if: inputs.component == 'sdk-go' && (inputs.task == 'test' || inputs.task == 'e2e') - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} files: reports/${{ inputs.task == 'test' && 'go-coverage.out' || 'go-coverage-e2e.out' }} @@ -165,7 +177,7 @@ jobs: - name: Upload Java coverage to Codecov if: inputs.component == 'sdk-java' && inputs.task == 'test' - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} files: reports/java-coverage/jacocoAggregated.xml @@ -177,7 +189,7 @@ jobs: - name: Upload C# coverage to Codecov if: inputs.component == 'sdk-csharp' && (inputs.task == 'test' || inputs.task == 'e2e') - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} files: foreign/csharp/reports/coverage.cobertura.xml diff --git a/.github/workflows/_test_bdd.yml b/.github/workflows/_test_bdd.yml index 75d762510e..f324bd4c72 100644 --- a/.github/workflows/_test_bdd.yml +++ b/.github/workflows/_test_bdd.yml @@ -38,7 +38,7 @@ jobs: timeout-minutes: 60 steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 - name: Skip noop if: inputs.component == 'noop' diff --git a/.github/workflows/_test_examples.yml b/.github/workflows/_test_examples.yml index 28942a81ee..0e105a075f 100644 --- a/.github/workflows/_test_examples.yml +++ b/.github/workflows/_test_examples.yml @@ -37,7 +37,7 @@ jobs: timeout-minutes: 60 steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 - name: Skip noop if: inputs.component == 'noop' @@ -63,7 +63,7 @@ jobs: - name: Setup uv if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python' - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - name: Install PHP build dependencies if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-php' diff --git a/.github/workflows/coverage-baseline.yml b/.github/workflows/coverage-baseline.yml index ccfd8f51a7..10c3124088 100644 --- a/.github/workflows/coverage-baseline.yml +++ b/.github/workflows/coverage-baseline.yml @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. -# Full coverage baseline for all 6 languages (Rust, Java, C#, Python, -# Node, Go). Runs on every push to master so Codecov has complete data +# Full coverage baseline for all 7 languages (Rust, Java, C#, Python, +# PHP, Node, Go). Runs on every push to master so Codecov has complete data # for carryforward on PR builds where only a subset of SDKs is tested. name: Coverage baseline @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 - name: Install system dependencies run: | @@ -54,9 +54,13 @@ jobs: with: # Also warms the GitHub Actions build cache for subsequent PR builds save-cache: "true" + # llvm-cov instrumentation roughly doubles object sizes; the light + # cleanup leaves too little headroom (build hit "No space left on + # device" at ~34 MB free). + free-disk-space-aggressive: "true" - name: Install cargo-llvm-cov - uses: taiki-e/install-action@v2.79.3 + uses: taiki-e/install-action@v2.81.8 with: tool: cargo-llvm-cov @@ -93,7 +97,7 @@ jobs: shell: bash - name: Upload to Codecov - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} files: codecov.json @@ -106,7 +110,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 - name: Setup Java with cache uses: ./.github/actions/utils/setup-java-with-cache @@ -136,7 +140,7 @@ jobs: log-file: ${{ steps.iggy.outputs.log_file }} - name: Upload to Codecov - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} files: foreign/java/build/reports/jacoco/aggregate/jacocoAggregated.xml @@ -149,10 +153,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 - name: Setup .NET - uses: actions/setup-dotnet@v5.2.0 + uses: actions/setup-dotnet@v5.3.0 with: dotnet-version: "10.0.x" @@ -208,7 +212,7 @@ jobs: dotnet-coverage merge ./reports/**/*.cobertura.xml -f cobertura -o ./reports/coverage.cobertura.xml - name: Upload to Codecov - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} files: foreign/csharp/reports/coverage.cobertura.xml @@ -221,7 +225,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 - name: Setup Python uses: actions/setup-python@v6 @@ -234,12 +238,12 @@ jobs: save-cache: "false" - name: Install cargo-llvm-cov - uses: taiki-e/install-action@v2.79.3 + uses: taiki-e/install-action@v2.81.8 with: tool: cargo-llvm-cov - name: Install uv - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - name: Install dependencies run: | @@ -297,10 +301,11 @@ jobs: sed -i 's|^SF:src/|SF:foreign/python/src/|' ../../reports/python-coverage.lcov echo "Coverage report generated: $(wc -l < ../../reports/python-coverage.lcov) lines" + ../../scripts/ci/validate-lcov.sh ../../reports/python-coverage.lcov shell: bash - name: Upload to Codecov - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} files: reports/python-coverage.lcov @@ -308,12 +313,33 @@ jobs: flags: python fail_ci_if_error: false + php-coverage: + name: PHP coverage baseline + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + + - name: Run PHP SDK tests with coverage + uses: ./.github/actions/php/pre-merge + with: + task: test + + - name: Upload to Codecov + uses: codecov/codecov-action@v7.0.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: reports/php-coverage.lcov + disable_search: true + flags: php + fail_ci_if_error: false + node-coverage: name: Node coverage baseline runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 - name: Setup Node.js uses: actions/setup-node@v6 @@ -364,7 +390,7 @@ jobs: log-file: ${{ steps.iggy.outputs.log_file }} - name: Upload to Codecov - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} files: reports/node-coverage/unit/lcov.info,reports/node-coverage/e2e/lcov.info @@ -377,7 +403,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 - name: Setup Go uses: ./.github/actions/utils/setup-go-with-cache @@ -438,7 +464,7 @@ jobs: log-file: ${{ steps.iggy.outputs.log_file }} - name: Upload to Codecov - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} files: reports/go-coverage.out @@ -451,7 +477,7 @@ jobs: runs-on: macos-14 timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 - name: Setup Rust with cache uses: ./.github/actions/utils/setup-rust-with-cache diff --git a/.github/workflows/edge-release.yml b/.github/workflows/edge-release.yml index 876c53cdc4..8c92c61a01 100644 --- a/.github/workflows/edge-release.yml +++ b/.github/workflows/edge-release.yml @@ -50,7 +50,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 - name: Get server version id: meta diff --git a/.github/workflows/issue-labeler-assigner.yml b/.github/workflows/issue-labeler-assigner.yml index 17de263e96..a1c3dab035 100644 --- a/.github/workflows/issue-labeler-assigner.yml +++ b/.github/workflows/issue-labeler-assigner.yml @@ -67,7 +67,7 @@ jobs: 'Python SDK': 'python', 'C# SDK': 'csharp', 'Node.js SDK': 'javascript', - 'C++ SDK': 'C++', + 'C++ SDK': 'cpp', 'PHP SDK': 'php', 'CLI': 'tui', 'Web UI': 'web', diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml index 6e45fa57e4..5de7a96ec1 100644 --- a/.github/workflows/post-merge.yml +++ b/.github/workflows/post-merge.yml @@ -47,7 +47,7 @@ jobs: crates_to_publish: ${{ steps.check.outputs.crates_to_publish }} sdks_to_publish: ${{ steps.check.outputs.sdks_to_publish }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 with: fetch-depth: 0 @@ -63,10 +63,10 @@ jobs: # cargo-rail computes the affected crate set for the Docker :edge gate. # Metadata-only `cargo rail plan` (no compile), so it runs on the runner's - # preinstalled cargo (rust-toolchain.toml pins 1.95.0) and skips the + # preinstalled cargo (rust-toolchain.toml pins 1.96.0) and skips the # heavyweight build-cache restore. - name: Install cargo-rail - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@v2.81.8 with: tool: cargo-rail diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 12b6a261ed..aa28ccd10c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -136,7 +136,7 @@ jobs: has_targets: ${{ steps.check.outputs.has_targets }} is_workflow_call: ${{ steps.detect.outputs.is_workflow_call }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 with: fetch-depth: 0 @@ -246,7 +246,7 @@ jobs: # policy contract intact. Composite action is shared with # .github/workflows/_common.yml. steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 with: ref: ${{ needs.validate.outputs.commit }} - uses: ./.github/actions/utils/validate-third-party-licenses @@ -276,7 +276,7 @@ jobs: chmod +x /tmp/copy-latest-from-master.sh echo "✅ Downloaded latest copy script from master" - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 with: ref: ${{ needs.validate.outputs.commit }} @@ -443,7 +443,7 @@ jobs: echo "✅ Downloaded latest copy script from master" - name: Checkout at commit - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 with: ref: ${{ needs.validate.outputs.commit }} # check-tags only runs `git ls-remote --tags` against origin, @@ -757,7 +757,7 @@ jobs: echo "✅ Downloaded latest copy script from master" - name: Checkout at commit - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 with: ref: ${{ needs.validate.outputs.commit }} fetch-depth: 0 @@ -851,7 +851,7 @@ jobs: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 with: ref: ${{ needs.validate.outputs.commit }} # create-git-tag's shallow-safe fallback at action.yml:86-96 @@ -941,7 +941,7 @@ jobs: path: ${{ runner.temp }}/digests - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Login to Docker Hub uses: ./.github/actions/utils/docker-login @@ -1073,7 +1073,7 @@ jobs: echo "✅ Downloaded latest copy script from master" - name: Checkout at commit - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 with: ref: ${{ needs.validate.outputs.commit }} # create-git-tag falls back to a shallow fetch when the commit @@ -1361,7 +1361,7 @@ jobs: chmod +x /tmp/copy-latest-from-master.sh echo "✅ Downloaded latest copy script from master" - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 with: ref: ${{ needs.validate.outputs.commit }} From dd96d1e36d062b705cfeb54e53531e1611a5d334 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Fri, 19 Jun 2026 21:18:55 -0400 Subject: [PATCH 15/19] Move resilience docs into README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Embed the resilience documentation (retry policy, circuit breaker, at-least-once delivery, and backfill behavior) from core/connectors/sources/opensearch_source/docs/RESILIENCE.md into the connector README. Remove the standalone RESILIENCE.md file and update internal references to point to the new Resilience section in README. No behavioral/code changes—documentation relocation and link updates only. --- .../sources/opensearch_source/README.md | 34 +++- .../opensearch_source/docs/RESILIENCE.md | 176 ------------------ 2 files changed, 31 insertions(+), 179 deletions(-) delete mode 100644 core/connectors/sources/opensearch_source/docs/RESILIENCE.md diff --git a/core/connectors/sources/opensearch_source/README.md b/core/connectors/sources/opensearch_source/README.md index ea3786af05..f17fd779bb 100644 --- a/core/connectors/sources/opensearch_source/README.md +++ b/core/connectors/sources/opensearch_source/README.md @@ -69,8 +69,6 @@ query = { match_all = {} } | `circuit_breaker_threshold` | `u32` | `5` | Consecutive poll failures before circuit opens | | `circuit_breaker_cool_down` | `String` | `"60s"` | Duration to skip polls when circuit is open | -See [docs/RESILIENCE.md](docs/RESILIENCE.md) for retry, circuit breaker, at-least-once, and backfill semantics. - ### File-backed state (optional) Runtime state is always returned from `poll()` and persisted by the connectors @@ -219,6 +217,35 @@ Requirements for correct operation: | `Storage` | Network or HTTP errors; client not initialized at `poll()` | | `Serialization` | JSON / MessagePack failures | +## Resilience + +### Retry policy + +| Category | Conditions | +| -------- | ---------- | +| Transient (retry) | Network errors; HTTP `429`; HTTP `5xx`; honors `Retry-After` header on `429` | +| Permanent (no retry) | `400`, `401`, `403`, `404`; malformed responses; search DSL errors | + +### Circuit breaker + +When open, `poll()` skips the search, logs a warning, sleeps `polling_interval`, and returns an empty batch with no state update - cursor does not advance. Consecutive failures after retries exhausted increment the breaker; a successful search resets it. + +### Delivery semantics + +**At-least-once toward Iggy.** The in-memory cursor advances in `finalize_poll()` before the runtime persists `ConnectorState`. A crash after cursor advance but before the runtime save re-emits the last batch on restart. + +Consumer guidance: dedup on OpenSearch `_id` plus index name (for index patterns), or a business key in `_source`. + +### Backfill + +Pagination is forward-only on `(timestamp_field asc, _id asc)`. Documents indexed with `timestamp_field` values older than the current cursor are not read until state is reset. + +| Mode | Use case | +| ---- | -------- | +| Forward tail (default) | Live streaming; cursor tracks ingest order | +| Bounded backfill | Set a time-window `query` in `plugin_config` | +| Full rescan | Reset runtime `ConnectorState`; duplicates possible | + ## Limitations - **Single sequential reader** — one `search_after` cursor, one batch per poll. @@ -233,10 +260,11 @@ Requirements for correct operation: parallel export. Offset paging (`from`/`size`) is not used. - **At-least-once delivery** — no deduplication by `_id`. The in-memory cursor advances before the runtime persists `ConnectorState`; a crash can re-emit the last batch. - See [docs/RESILIENCE.md](docs/RESILIENCE.md). + See [Resilience](#resilience). - **HTTP retry and circuit breaker** — transient `429`/`5xx` and network errors are retried per `max_retries`; consecutive failures trip a circuit breaker that skips polls until cool-down. Permanent errors (`4xx` except `429`) are not retried. + See [Resilience](#resilience). - **Backfill gap** — documents indexed with `timestamp_field` values older than the current cursor are not read until connector state is reset. - **Full `_source` only** — entire document JSON is published; no field diff --git a/core/connectors/sources/opensearch_source/docs/RESILIENCE.md b/core/connectors/sources/opensearch_source/docs/RESILIENCE.md deleted file mode 100644 index cad7a3a9da..0000000000 --- a/core/connectors/sources/opensearch_source/docs/RESILIENCE.md +++ /dev/null @@ -1,176 +0,0 @@ -# OpenSearch Source — Resilience & Delivery Semantics - -Design reference for HTTP retry, circuit breaker, at-least-once delivery, and -backfill behavior. Phase 1 (retry + circuit breaker) is implemented in the -connector; later phases are documented for future work. - -## Contents - -- [HTTP retry and circuit breaker](#http-retry-and-circuit-breaker) -- [At-least-once delivery](#at-least-once-delivery) -- [Backfill gap](#backfill-gap) -- [Implementation phases](#implementation-phases) -- [Non-goals](#non-goals) - -## HTTP retry and circuit breaker - -### Problem - -Each poll performs HTTP calls to OpenSearch (`HEAD` index exists at `open()`, -`POST /{index}/_search` at `poll()`). Without retry, transient `503`, `429`, or -network blips fail the poll immediately. The connector only retries on the next -`polling_interval` cycle. - -### Constraint - -InfluxDB uses `reqwest` + `build_retry_client` middleware. This connector uses -`opensearch-rs` (`TransportBuilder`). Retry is implemented at the **plugin level** -using `iggy_connector_sdk::retry` helpers (`exponential_backoff`, `jitter`, -`parse_retry_after`). - -### Configuration (`plugin_config`) - -| Field | Default | Purpose | -| ----- | ------- | ------- | -| `max_retries` | `3` | Total attempts per HTTP operation during `poll()` | -| `retry_delay` | `1s` | Base backoff between attempts | -| `retry_max_delay` | `30s` | Cap per retry wait | -| `max_open_retries` | `5` | Startup attempts for index-exists check in `open()` | -| `open_retry_max_delay` | `30s` | Cap for open probe backoff | -| `circuit_breaker_threshold` | `5` | Consecutive failures before circuit opens | -| `circuit_breaker_cool_down` | `60s` | Circuit open duration | - -All fields are optional (`#[serde(default)]`); omitted keys use the defaults above. - -### Retry policy - -**Transient (retry):** network errors; HTTP `429`; HTTP `5xx`; `Retry-After` on `429` -when parseable. - -**Permanent (no retry):** `400`, `401`, `403`, `404`; malformed responses; -search DSL errors. - -### Circuit breaker - -When open, `poll()` skips the search, logs a warning, sleeps `polling_interval`, -and returns an empty batch with `state: None` so the runtime does not persist a -new cursor (same pattern as InfluxDB source). - -Consecutive poll failures (after retries exhausted) increment the breaker. -Successful search resets it. - -## At-least-once delivery - -### Contract - -The connector provides **at-least-once** delivery toward Iggy. Duplicates can -occur when: - -- The process crashes after `finalize_poll` advances the in-memory cursor but - before the runtime persists `ConnectorState`. -- The connector restarts from a cursor behind the last published batch. - -The runtime saves `ConnectorState` only after a successful Iggy send. - -### Mitigations (future phases) - -| Tier | Action | Status | -| ---- | ------ | ------ | -| A | Document contract + recommend consumer dedup on `_id` | Partial (README) | -| B | Set `ProducedMessage.id` from OpenSearch `_id` | Planned | -| C | Runtime ack before cursor advance | Deferred (FFI change) | - -### Consumer guidance - -- Dedup on OpenSearch `_id` plus index name, or a business key in `_source`. -- Treat replays as normal for log pipelines; use upsert sinks where needed. - -## Backfill gap - -### Contract - -Pagination is forward-only on `(timestamp_field asc, _id asc)` via `search_after`. -After catch-up, documents indexed with `timestamp_field` **older** than the -current cursor are not read until connector state is reset. - -### Operator modes - -| Mode | Use case | -| ---- | -------- | -| Forward tail (default) | Live streaming; timestamps track ingest order | -| Bounded backfill | One-time load via time-window `query` | -| Full rescan | Reset runtime `ConnectorState` (duplicates possible) | - -### Future phases - -| Tier | Feature | -| ---- | ------- | -| B | `backfill.mode = warn` — log when index min timestamp < cursor | -| C | `backfill.mode = window` — explicit time-window scan | -| E | `min_timestamp` on fresh start only | - -## Implementation phases - -| Phase | Scope | Status | -| ----- | ----- | ------ | -| 1 | HTTP retry + circuit breaker | **Implemented** | -| 2 | At-least-once + backfill runbook in README | Partial | -| 3 | `ProducedMessage.id` from `_id`; backfill warn mode | Planned | -| 4 | Backfill window mode | Planned | -| 5 | Exactly-once / runtime ack | Deferred | - -## Testing retry and circuit breaker - -Two layers cover resilience behavior. Use both: plugin tests are fast and precise; -integration tests exercise the full connectors runtime + FFI poll loop. - -### Plugin HTTP tests (fast, no Docker) - -In-process axum mock server in `src/http_tests.rs`. Run: - -```bash -cargo test -p iggy_connector_opensearch_source given_transient_search_errors_when_poll_should_retry_and_succeed -cargo test -p iggy_connector_opensearch_source given_circuit_breaker_open_when_poll_should_return_empty_without_error -``` - -These tests inject `503`/`500` responses and assert retry counts and circuit-breaker -skip behavior at the plugin `poll()` boundary. - -### Integration tests (runtime + wiremock) - -Fixtures in `core/integration/tests/connectors/fixtures/opensearch/resilience.rs` -start a [wiremock](https://github.com/LukeMathWalker/wiremock-rs) `MockServer` (no -OpenSearch container) and point the connector URL at it via runtime env overrides: - -| Fixture | Behavior | -| ------- | -------- | -| `OpenSearchSourceTransientErrorFixture` | Two `503` search responses, then one hit | -| `OpenSearchSourceCircuitBreakerFixture` | Persistent `500` with fast retry settings | - -Circuit-breaker **open/skip** semantics (empty poll, no cursor advance) are asserted in -plugin `http_tests.rs` because they are precise to time and poll count. The integration -fixture verifies the runtime keeps the source `Running`, performs HTTP retries, and does -not publish messages under sustained failures. - -Run: - -```bash -cargo test -p integration -- connectors::opensearch::opensearch_source_resilience -``` - -Tests live in `opensearch_source_resilience.rs` and verify message production after -retry, plus runtime `Running` status with no messages under sustained failures. - -### Manual / staging checks - -Point `url` at a proxy or fault-injection layer in front of a real cluster. Tune -`max_retries`, `retry_delay`, `circuit_breaker_threshold`, and `circuit_breaker_cool_down` -in `plugin_config`. Watch connector logs for `OpenSearch request failed; retrying` and -`circuit breaker is OPEN`. - -## Non-goals - -- Exactly-once delivery in the connector alone -- Parallel slice / PIT export (separate design) -- Automatic full-index rescan without operator intent -- Replacing `opensearch-rs` with raw `reqwest` unless plugin retry proves insufficient From fbcc4f868954022955ccb9851ea468b4731ea9cc Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Fri, 19 Jun 2026 23:24:32 -0400 Subject: [PATCH 16/19] Add Apache 2.0 license headers to OpenSearch files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Apache License, Version 2.0 header comments to various OpenSearch source and test files for licensing compliance. Affected paths include core/connectors/sources/opensearch_source (lib, retry, state_manager, http_tests) and multiple integration test fixtures and opensearch test files under core/integration/tests/connectors/opensearch. No functional changes — only file header updates to ensure consistent license attribution. --- .../sources/opensearch_source/src/http_tests.rs | 17 +++++++++++++++++ .../sources/opensearch_source/src/lib.rs | 17 +++++++++++++++++ .../sources/opensearch_source/src/retry.rs | 17 +++++++++++++++++ .../opensearch_source/src/state_manager.rs | 17 +++++++++++++++++ .../connectors/fixtures/opensearch/container.rs | 17 +++++++++++++++++ .../tests/connectors/fixtures/opensearch/mod.rs | 17 +++++++++++++++++ .../fixtures/opensearch/resilience.rs | 17 +++++++++++++++++ .../connectors/fixtures/opensearch/source.rs | 17 +++++++++++++++++ .../tests/connectors/opensearch/mod.rs | 17 +++++++++++++++++ .../connectors/opensearch/opensearch_source.rs | 17 +++++++++++++++++ .../opensearch/opensearch_source_resilience.rs | 17 +++++++++++++++++ .../opensearch/opensearch_source_types.rs | 17 +++++++++++++++++ 12 files changed, 204 insertions(+) diff --git a/core/connectors/sources/opensearch_source/src/http_tests.rs b/core/connectors/sources/opensearch_source/src/http_tests.rs index 901ca73153..641466739c 100644 --- a/core/connectors/sources/opensearch_source/src/http_tests.rs +++ b/core/connectors/sources/opensearch_source/src/http_tests.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs index 927c96a2eb..124e0ed693 100644 --- a/core/connectors/sources/opensearch_source/src/lib.rs +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file diff --git a/core/connectors/sources/opensearch_source/src/retry.rs b/core/connectors/sources/opensearch_source/src/retry.rs index 9e947ae5f8..19f63670dc 100644 --- a/core/connectors/sources/opensearch_source/src/retry.rs +++ b/core/connectors/sources/opensearch_source/src/retry.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file diff --git a/core/connectors/sources/opensearch_source/src/state_manager.rs b/core/connectors/sources/opensearch_source/src/state_manager.rs index 5e980246c9..bdcdc3bcbf 100644 --- a/core/connectors/sources/opensearch_source/src/state_manager.rs +++ b/core/connectors/sources/opensearch_source/src/state_manager.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file diff --git a/core/integration/tests/connectors/fixtures/opensearch/container.rs b/core/integration/tests/connectors/fixtures/opensearch/container.rs index 81440b7aa8..b11054509f 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/container.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/container.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file diff --git a/core/integration/tests/connectors/fixtures/opensearch/mod.rs b/core/integration/tests/connectors/fixtures/opensearch/mod.rs index d690e8a4cf..c03a6ed30f 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/mod.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/mod.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file diff --git a/core/integration/tests/connectors/fixtures/opensearch/resilience.rs b/core/integration/tests/connectors/fixtures/opensearch/resilience.rs index a107737111..14fb2679df 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/resilience.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/resilience.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file diff --git a/core/integration/tests/connectors/fixtures/opensearch/source.rs b/core/integration/tests/connectors/fixtures/opensearch/source.rs index 80f51f7b8f..84d6b29d7a 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/source.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/source.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file diff --git a/core/integration/tests/connectors/opensearch/mod.rs b/core/integration/tests/connectors/opensearch/mod.rs index 656ab91645..f7d9c23914 100644 --- a/core/integration/tests/connectors/opensearch/mod.rs +++ b/core/integration/tests/connectors/opensearch/mod.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file diff --git a/core/integration/tests/connectors/opensearch/opensearch_source.rs b/core/integration/tests/connectors/opensearch/opensearch_source.rs index 2da0ed8545..41cc0a9b80 100644 --- a/core/integration/tests/connectors/opensearch/opensearch_source.rs +++ b/core/integration/tests/connectors/opensearch/opensearch_source.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file diff --git a/core/integration/tests/connectors/opensearch/opensearch_source_resilience.rs b/core/integration/tests/connectors/opensearch/opensearch_source_resilience.rs index c4577b55eb..9c76884453 100644 --- a/core/integration/tests/connectors/opensearch/opensearch_source_resilience.rs +++ b/core/integration/tests/connectors/opensearch/opensearch_source_resilience.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file diff --git a/core/integration/tests/connectors/opensearch/opensearch_source_types.rs b/core/integration/tests/connectors/opensearch/opensearch_source_types.rs index 1848137ae7..e700578c89 100644 --- a/core/integration/tests/connectors/opensearch/opensearch_source_types.rs +++ b/core/integration/tests/connectors/opensearch/opensearch_source_types.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file From 553e7a637be9e9520dce567ec1e40a2fd04872b4 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Fri, 19 Jun 2026 23:37:07 -0400 Subject: [PATCH 17/19] Remove duplicated Apache license headers Strip redundant Apache Software Foundation license block comments from OpenSearch source and integration test files to clean up file headers. Affected files include core/connectors/sources/opensearch_source/src/{http_tests.rs,lib.rs,retry.rs,state_manager.rs} and several integration test fixtures and opensearch tests under core/integration/tests/connectors/{fixtures/opensearch,opensearch}. No functional code changes. --- .../opensearch_source/src/http_tests.rs | 19 ------------------- .../sources/opensearch_source/src/lib.rs | 18 ------------------ .../sources/opensearch_source/src/retry.rs | 19 ------------------- .../opensearch_source/src/state_manager.rs | 19 ------------------- .../fixtures/opensearch/container.rs | 19 ------------------- .../connectors/fixtures/opensearch/mod.rs | 19 ------------------- .../fixtures/opensearch/resilience.rs | 19 ------------------- .../connectors/fixtures/opensearch/source.rs | 19 ------------------- .../tests/connectors/opensearch/mod.rs | 19 ------------------- .../opensearch/opensearch_source.rs | 19 ------------------- .../opensearch_source_resilience.rs | 19 ------------------- .../opensearch/opensearch_source_types.rs | 19 ------------------- 12 files changed, 227 deletions(-) diff --git a/core/connectors/sources/opensearch_source/src/http_tests.rs b/core/connectors/sources/opensearch_source/src/http_tests.rs index 641466739c..52efad06a0 100644 --- a/core/connectors/sources/opensearch_source/src/http_tests.rs +++ b/core/connectors/sources/opensearch_source/src/http_tests.rs @@ -15,25 +15,6 @@ // specific language governing permissions and limitations // under the License. -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - use crate::state_manager::{SourceState, create_state_storage}; use crate::{OpenSearchSource, OpenSearchSourceConfig, StateConfig}; use axum::Router; diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs index 124e0ed693..5fd24b6724 100644 --- a/core/connectors/sources/opensearch_source/src/lib.rs +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -15,24 +15,6 @@ // specific language governing permissions and limitations // under the License. -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ use async_trait::async_trait; use iggy_common::{DateTime, Utc}; use iggy_connector_sdk::retry::{CircuitBreaker, parse_duration}; diff --git a/core/connectors/sources/opensearch_source/src/retry.rs b/core/connectors/sources/opensearch_source/src/retry.rs index 19f63670dc..587b3dfbb2 100644 --- a/core/connectors/sources/opensearch_source/src/retry.rs +++ b/core/connectors/sources/opensearch_source/src/retry.rs @@ -15,25 +15,6 @@ // specific language governing permissions and limitations // under the License. -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - use iggy_connector_sdk::retry::{exponential_backoff, jitter, parse_retry_after}; use std::time::Duration; use tokio::time::sleep; diff --git a/core/connectors/sources/opensearch_source/src/state_manager.rs b/core/connectors/sources/opensearch_source/src/state_manager.rs index bdcdc3bcbf..a11bb91211 100644 --- a/core/connectors/sources/opensearch_source/src/state_manager.rs +++ b/core/connectors/sources/opensearch_source/src/state_manager.rs @@ -15,25 +15,6 @@ // specific language governing permissions and limitations // under the License. -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - use crate::{OpenSearchSource, StateConfig}; use async_trait::async_trait; use iggy_common::{DateTime, Utc}; diff --git a/core/integration/tests/connectors/fixtures/opensearch/container.rs b/core/integration/tests/connectors/fixtures/opensearch/container.rs index b11054509f..c4518d8a5a 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/container.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/container.rs @@ -15,25 +15,6 @@ // specific language governing permissions and limitations // under the License. -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - use crate::connectors::fixtures; use integration::harness::TestBinaryError; use reqwest_middleware::ClientWithMiddleware as HttpClient; diff --git a/core/integration/tests/connectors/fixtures/opensearch/mod.rs b/core/integration/tests/connectors/fixtures/opensearch/mod.rs index c03a6ed30f..c943aa06b2 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/mod.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/mod.rs @@ -15,25 +15,6 @@ // specific language governing permissions and limitations // under the License. -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - pub mod container; pub mod resilience; pub mod source; diff --git a/core/integration/tests/connectors/fixtures/opensearch/resilience.rs b/core/integration/tests/connectors/fixtures/opensearch/resilience.rs index 14fb2679df..e6268197c2 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/resilience.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/resilience.rs @@ -15,25 +15,6 @@ // specific language governing permissions and limitations // under the License. -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - use super::container::{ DEFAULT_TEST_STREAM, DEFAULT_TEST_TOPIC, ENV_SOURCE_BATCH_SIZE, ENV_SOURCE_INDEX, ENV_SOURCE_MAX_RETRIES, ENV_SOURCE_OPEN_RETRY_MAX_DELAY, ENV_SOURCE_PATH, diff --git a/core/integration/tests/connectors/fixtures/opensearch/source.rs b/core/integration/tests/connectors/fixtures/opensearch/source.rs index 84d6b29d7a..b2f2cfffcb 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/source.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/source.rs @@ -15,25 +15,6 @@ // specific language governing permissions and limitations // under the License. -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - use super::container::{ DEFAULT_TEST_STREAM, DEFAULT_TEST_TOPIC, ENV_SOURCE_BATCH_SIZE, ENV_SOURCE_INDEX, ENV_SOURCE_PATH, ENV_SOURCE_POLLING_INTERVAL, ENV_SOURCE_STREAMS_0_SCHEMA, diff --git a/core/integration/tests/connectors/opensearch/mod.rs b/core/integration/tests/connectors/opensearch/mod.rs index f7d9c23914..394a024035 100644 --- a/core/integration/tests/connectors/opensearch/mod.rs +++ b/core/integration/tests/connectors/opensearch/mod.rs @@ -15,25 +15,6 @@ // specific language governing permissions and limitations // under the License. -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - mod opensearch_source; mod opensearch_source_resilience; mod opensearch_source_types; diff --git a/core/integration/tests/connectors/opensearch/opensearch_source.rs b/core/integration/tests/connectors/opensearch/opensearch_source.rs index 41cc0a9b80..b385e155af 100644 --- a/core/integration/tests/connectors/opensearch/opensearch_source.rs +++ b/core/integration/tests/connectors/opensearch/opensearch_source.rs @@ -15,25 +15,6 @@ // specific language governing permissions and limitations // under the License. -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - use super::{POLL_ATTEMPTS, POLL_INTERVAL_MS, TEST_MESSAGE_COUNT}; use crate::connectors::fixtures::{ OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture, diff --git a/core/integration/tests/connectors/opensearch/opensearch_source_resilience.rs b/core/integration/tests/connectors/opensearch/opensearch_source_resilience.rs index 9c76884453..aaf1bbb7b0 100644 --- a/core/integration/tests/connectors/opensearch/opensearch_source_resilience.rs +++ b/core/integration/tests/connectors/opensearch/opensearch_source_resilience.rs @@ -15,25 +15,6 @@ // specific language governing permissions and limitations // under the License. -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - use super::{POLL_ATTEMPTS, POLL_INTERVAL_MS}; use crate::connectors::fixtures::{ OpenSearchSourceCircuitBreakerFixture, OpenSearchSourceTransientErrorFixture, diff --git a/core/integration/tests/connectors/opensearch/opensearch_source_types.rs b/core/integration/tests/connectors/opensearch/opensearch_source_types.rs index e700578c89..45051a25a0 100644 --- a/core/integration/tests/connectors/opensearch/opensearch_source_types.rs +++ b/core/integration/tests/connectors/opensearch/opensearch_source_types.rs @@ -15,25 +15,6 @@ // specific language governing permissions and limitations // under the License. -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - use super::{POLL_ATTEMPTS, POLL_INTERVAL_MS, TEST_MESSAGE_COUNT}; use crate::connectors::fixtures::{ OpenSearchSourcePreCreatedFixture, OpenSearchSourceTypedFieldsFixture, From 9d1254e79593ab4c2f4fa7256225d0b889f2dd05 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Fri, 19 Jun 2026 23:41:54 -0400 Subject: [PATCH 18/19] Format closure in OpenSearchSource init Expand the ok_or_else closure into a block for clearer formatting in core/connectors/sources/opensearch_source/src/lib.rs. This is a non-functional change that improves readability around the connector initialization error message. --- core/connectors/sources/opensearch_source/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs index 5fd24b6724..63002b3542 100644 --- a/core/connectors/sources/opensearch_source/src/lib.rs +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -457,7 +457,9 @@ impl OpenSearchSource { let mut search_body = self .search_body_base .as_ref() - .ok_or_else(|| Error::Connection("connector not initialized; call open() first".to_string()))? + .ok_or_else(|| { + Error::Connection("connector not initialized; call open() first".to_string()) + })? .clone(); if let Some(cursor) = search_after { From f8b7e7b2f96cd44bd5dcb542826bb6cd20c3f9d2 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 00:12:05 -0400 Subject: [PATCH 19/19] Add circuit-breaker envs; simplify timestamp check Use an inclusive range contains check when deciding if a numeric timestamp is already in milliseconds (improves readability and avoids manual comparisons). Update OpenSearch resilience test fixture: import new env constants, set ENV_SOURCE_MAX_OPEN_RETRIES to "2", and configure circuit breaker envs (ENV_SOURCE_CIRCUIT_BREAKER_THRESHOLD="3", ENV_SOURCE_CIRCUIT_BREAKER_COOL_DOWN="500ms") so tests exercise the circuit-breaker behavior. --- .../sources/opensearch_source/src/lib.rs | 2 +- .../fixtures/opensearch/resilience.rs | 23 ++++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs index 63002b3542..2a4ffbc365 100644 --- a/core/connectors/sources/opensearch_source/src/lib.rs +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -714,7 +714,7 @@ fn parse_document_timestamp(value: &Value) -> Option> { // Values above 1e12 are already milliseconds (Unix epoch seconds won't reach // 1e12 until year 33658). Values at or below are treated as seconds and // multiplied by 1000. - let millis = if raw > 1_000_000_000_000 || raw < -1_000_000_000_000 { + let millis = if !(-1_000_000_000_000..=1_000_000_000_000).contains(&raw) { raw } else { raw.saturating_mul(1_000) diff --git a/core/integration/tests/connectors/fixtures/opensearch/resilience.rs b/core/integration/tests/connectors/fixtures/opensearch/resilience.rs index e6268197c2..694b27ab81 100644 --- a/core/integration/tests/connectors/fixtures/opensearch/resilience.rs +++ b/core/integration/tests/connectors/fixtures/opensearch/resilience.rs @@ -16,11 +16,12 @@ // under the License. use super::container::{ - DEFAULT_TEST_STREAM, DEFAULT_TEST_TOPIC, ENV_SOURCE_BATCH_SIZE, ENV_SOURCE_INDEX, - ENV_SOURCE_MAX_RETRIES, ENV_SOURCE_OPEN_RETRY_MAX_DELAY, ENV_SOURCE_PATH, - ENV_SOURCE_POLLING_INTERVAL, ENV_SOURCE_RETRY_DELAY, ENV_SOURCE_RETRY_MAX_DELAY, - ENV_SOURCE_STREAMS_0_SCHEMA, ENV_SOURCE_STREAMS_0_STREAM, ENV_SOURCE_STREAMS_0_TOPIC, - ENV_SOURCE_TIMESTAMP_FIELD, ENV_SOURCE_URL, + DEFAULT_TEST_STREAM, DEFAULT_TEST_TOPIC, ENV_SOURCE_BATCH_SIZE, + ENV_SOURCE_CIRCUIT_BREAKER_COOL_DOWN, ENV_SOURCE_CIRCUIT_BREAKER_THRESHOLD, ENV_SOURCE_INDEX, + ENV_SOURCE_MAX_OPEN_RETRIES, ENV_SOURCE_MAX_RETRIES, ENV_SOURCE_OPEN_RETRY_MAX_DELAY, + ENV_SOURCE_PATH, ENV_SOURCE_POLLING_INTERVAL, ENV_SOURCE_RETRY_DELAY, + ENV_SOURCE_RETRY_MAX_DELAY, ENV_SOURCE_STREAMS_0_SCHEMA, ENV_SOURCE_STREAMS_0_STREAM, + ENV_SOURCE_STREAMS_0_TOPIC, ENV_SOURCE_TIMESTAMP_FIELD, ENV_SOURCE_URL, }; use async_trait::async_trait; use integration::harness::{TestBinaryError, TestFixture}; @@ -84,6 +85,7 @@ fn resilience_base_envs(mock_uri: &str) -> HashMap { ENV_SOURCE_OPEN_RETRY_MAX_DELAY.to_string(), "200ms".to_string(), ); + envs.insert(ENV_SOURCE_MAX_OPEN_RETRIES.to_string(), "2".to_string()); envs } @@ -188,6 +190,15 @@ impl TestFixture for OpenSearchSourceCircuitBreakerFixture { } fn connectors_runtime_envs(&self) -> HashMap { - resilience_base_envs(&self.mock_server.uri()) + let mut envs = resilience_base_envs(&self.mock_server.uri()); + envs.insert( + ENV_SOURCE_CIRCUIT_BREAKER_THRESHOLD.to_string(), + "3".to_string(), + ); + envs.insert( + ENV_SOURCE_CIRCUIT_BREAKER_COOL_DOWN.to_string(), + "500ms".to_string(), + ); + envs } }