Skip to content

Resolve OpenAPI drift: object storage ingestion controls, pg autovacuum config, horizontal autoscaling fields (#287) #288

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sdairs merged 1 commit into from
Jul 15, 2026
Merged

Resolve OpenAPI drift: object storage ingestion controls, pg autovacuum config, horizontal autoscaling fields (#287) #288

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
243 changes: 218 additions & 25 deletions crates/clickhouse-cloud-api/clickhouse_cloud_openapi.json
View file
Open in desktop

Large diffs are not rendered by default.

42 changes: 42 additions & 0 deletions crates/clickhouse-cloud-api/src/models.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8391,6 +8391,10 @@ pub struct ClickPipeObjectStorageSource {
pub path: Option<String>,
#[serde(rename = "queueUrl", skip_serializing_if = "Option::is_none", default)]
pub queue_url: Option<String>,
#[serde(rename = "skipInitialLoad", skip_serializing_if = "Option::is_none", default)]
pub skip_initial_load: Option<bool>,
#[serde(rename = "startAfter", skip_serializing_if = "Option::is_none", default)]
pub start_after: Option<String>,
#[serde(default)]
pub r#type: ClickPipeObjectStorageSourceType,
#[serde(default)]
Expand Down Expand Up @@ -8546,6 +8550,10 @@ pub struct ClickPipePatchObjectStorageSource {
pub path: Option<String>,
#[serde(rename = "serviceAccountKey", skip_serializing_if = "Option::is_none", default)]
pub service_account_key: Option<String>,
#[serde(rename = "skipInitialLoad", skip_serializing_if = "Option::is_none", default)]
pub skip_initial_load: Option<bool>,
#[serde(rename = "startAfter", skip_serializing_if = "Option::is_none", default)]
pub start_after: Option<String>,
}

/// `ClickPipePatchPostgresPipeRemoveTableMapping` from the ClickHouse Cloud API.
Expand Down Expand Up @@ -8726,6 +8734,10 @@ pub struct ClickPipePostObjectStorageSource {
pub queue_url: Option<String>,
#[serde(rename = "serviceAccountKey", skip_serializing_if = "Option::is_none", default)]
pub service_account_key: Option<String>,
#[serde(rename = "skipInitialLoad", skip_serializing_if = "Option::is_none", default)]
pub skip_initial_load: Option<bool>,
#[serde(rename = "startAfter", skip_serializing_if = "Option::is_none", default)]
pub start_after: Option<String>,
#[serde(default)]
pub r#type: ClickPipePostObjectStorageSourceType,
#[serde(default)]
Expand Down Expand Up @@ -10981,6 +10993,10 @@ pub struct ScalingScheduleEntry {
pub min_replicas: Option<i64>,
#[serde(default)]
pub name: String,
#[serde(rename = "numReplicas", skip_serializing_if = "Option::is_none", default)]
pub num_replicas: Option<i64>,
#[serde(rename = "replicaMemoryGb", skip_serializing_if = "Option::is_none", default)]
pub replica_memory_gb: Option<f64>,
#[serde(rename = "startHourUtc", default)]
pub start_hour_utc: i64,
#[serde(default)]
Expand All @@ -11006,6 +11022,10 @@ pub struct ScalingScheduleEntryRequest {
pub min_replicas: Option<i64>,
#[serde(default)]
pub name: String,
#[serde(rename = "numReplicas", skip_serializing_if = "Option::is_none", default)]
pub num_replicas: Option<i64>,
#[serde(rename = "replicaMemoryGb", skip_serializing_if = "Option::is_none", default)]
pub replica_memory_gb: Option<f64>,
#[serde(rename = "startHourUtc", default)]
pub start_hour_utc: i64,
#[serde(default)]
Expand Down Expand Up @@ -11873,11 +11893,15 @@ pub struct ServicePostRequest {
pub is_readonly: Option<bool>,
#[serde(rename = "maxReplicaMemoryGb", skip_serializing_if = "Option::is_none", default)]
pub max_replica_memory_gb: Option<f64>,
#[serde(rename = "maxReplicas", skip_serializing_if = "Option::is_none", default)]
pub max_replicas: Option<f64>,
#[cfg(feature = "deprecated-fields")]
#[serde(rename = "maxTotalMemoryGb", skip_serializing_if = "Option::is_none", default)]
pub max_total_memory_gb: Option<f64>,
#[serde(rename = "minReplicaMemoryGb", skip_serializing_if = "Option::is_none", default)]
pub min_replica_memory_gb: Option<f64>,
#[serde(rename = "minReplicas", skip_serializing_if = "Option::is_none", default)]
pub min_replicas: Option<f64>,
#[cfg(feature = "deprecated-fields")]
#[serde(rename = "minTotalMemoryGb", skip_serializing_if = "Option::is_none", default)]
pub min_total_memory_gb: Option<f64>,
Expand All @@ -11898,6 +11922,8 @@ pub struct ServicePostRequest {
pub region: ServicePostRequestRegion,
#[serde(rename = "releaseChannel", skip_serializing_if = "Option::is_none", default)]
pub release_channel: Option<ServicePostRequestReleasechannel>,
#[serde(rename = "replicaMemoryGb", skip_serializing_if = "Option::is_none", default)]
pub replica_memory_gb: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub tags: Option<Vec<ResourceTagsV1>>,
#[cfg(feature = "deprecated-fields")]
Expand Down Expand Up @@ -12150,6 +12176,22 @@ pub struct PgBouncerConfig {
/// `pgConfig` from the ClickHouse Cloud API.
#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
pub struct PgConfig {
#[serde(skip_serializing_if = "Option::is_none", default)]
pub autovacuum_analyze_scale_factor: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub autovacuum_max_workers: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub autovacuum_naptime: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub autovacuum_vacuum_cost_delay: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub autovacuum_vacuum_cost_limit: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub autovacuum_vacuum_insert_scale_factor: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub autovacuum_vacuum_scale_factor: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub autovacuum_work_mem: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub default_transaction_isolation: Option<PgConfigDefaultTransactionIsolation>,
#[serde(skip_serializing_if = "Option::is_none", default)]
Expand Down
2 changes: 2 additions & 0 deletions crates/clickhouse-cloud-api/tests/common/support.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -914,6 +914,8 @@ pub fn scaling_schedule_entry_to_request(
min_replica_memory_gb: entry.min_replica_memory_gb,
min_replicas: entry.min_replicas,
name: entry.name.clone(),
num_replicas: entry.num_replicas,
replica_memory_gb: entry.replica_memory_gb,
start_hour_utc: entry.start_hour_utc,
weekdays: entry.weekdays.clone(),
}
Expand Down
39 changes: 31 additions & 8 deletions crates/clickhouse-cloud-api/tests/integration_postgres_test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,8 +230,9 @@ async fn cloud_postgres_crud_lifecycle() -> TestResult<()> {
.await?;
}

// Round-trip a single pgConfig field: PATCH max_connections to a
// new value, poll-until GET reflects it, then PATCH back. GET
// Round-trip pgConfig fields: PATCH max_connections and
// autovacuum_max_workers to new values, poll-until GET reflects
// them, then PATCH back. GET
// returns numeric pgConfig values wrapped in JSON strings (the spec
// types them string-or-number), so extract via pg_config_value_as_i64
// and compare tolerantly.
Expand All @@ -243,12 +244,21 @@ async fn cloud_postgres_crud_lifecycle() -> TestResult<()> {
.and_then(pg_config_value_as_i64)
.unwrap_or(100);
let target = baseline_max + 7;
// Postgres defaults autovacuum_max_workers to 3; the per-run
// service is deleted at the end, so drift on reset is harmless.
let baseline_autovacuum = baseline
.pg_config
.autovacuum_max_workers
.as_ref()
.and_then(pg_config_value_as_i64)
.unwrap_or(3);
let autovacuum_target = baseline_autovacuum + 1;

failures
.run(
&ctx,
StepKind::NonBlocking,
"patch pgConfig.max_connections",
"patch pgConfig.max_connections/autovacuum_max_workers",
|| {
let client = client.clone();
let org_id = ctx.org_id.clone();
Expand All @@ -257,6 +267,9 @@ async fn cloud_postgres_crud_lifecycle() -> TestResult<()> {
let body = PostgresInstanceConfig {
pg_config: PgConfig {
max_connections: Some(serde_json::json!(target)),
autovacuum_max_workers: Some(serde_json::json!(
autovacuum_target
)),
..Default::default()
},
pg_bouncer_config: PgBouncerConfig::default(),
Expand All @@ -283,7 +296,7 @@ async fn cloud_postgres_crud_lifecycle() -> TestResult<()> {
let interval = ctx.poll_interval;
async move {
poll_until(
"pg_config.max_connections == target",
"pg_config.max_connections/autovacuum_max_workers == targets",
timeout,
interval,
|| {
Expand All @@ -294,12 +307,19 @@ async fn cloud_postgres_crud_lifecycle() -> TestResult<()> {
let resp = client
.postgres_instance_config_get(&org_id, &postgres_id)
.await?;
let observed = resp
.result
.and_then(|r| r.pg_config.max_connections)
let pg_config =
resp.result.map(|r| r.pg_config).unwrap_or_default();
let observed = pg_config
.max_connections
.as_ref()
.and_then(pg_config_value_as_i64);
if observed == Some(target) {
let observed_autovacuum = pg_config
.autovacuum_max_workers
.as_ref()
.and_then(pg_config_value_as_i64);
if observed == Some(target)
&& observed_autovacuum == Some(autovacuum_target)
{
Ok(Some(()))
} else {
Ok(None)
Expand All @@ -326,6 +346,9 @@ async fn cloud_postgres_crud_lifecycle() -> TestResult<()> {
let body = PostgresInstanceConfig {
pg_config: PgConfig {
max_connections: Some(serde_json::json!(baseline_max)),
autovacuum_max_workers: Some(serde_json::json!(
baseline_autovacuum
)),
..Default::default()
},
pg_bouncer_config: PgBouncerConfig::default(),
Expand Down
4 changes: 4 additions & 0 deletions crates/clickhouse-cloud-api/tests/integration_test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2032,6 +2032,10 @@ async fn cloud_service_crud_lifecycle() -> TestResult<()> {
max_replicas: Some(base_replicas as i64),
idle_scaling: Some(true),
idle_timeout_minutes: Some(5),
// Vertical entry expressed as equal min/max; the fixed-count
// and horizontal-autoscaling forms need org-level enablement.
num_replicas: None,
replica_memory_gb: None,
};

let upserted = failures
Expand Down
83 changes: 82 additions & 1 deletion crates/clickhouse-cloud-api/tests/models_test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,28 @@ fn serialize_service_post_request() {
assert_eq!(json["ipAccessList"][0]["source"], "0.0.0.0/0");
}

#[test]
fn serialize_service_post_request_horizontal_autoscaling() {
let req = ServicePostRequest {
name: "horizontal-service".to_string(),
min_replicas: Some(1.0),
max_replicas: Some(5.0),
replica_memory_gb: Some(32.0),
..Default::default()
};
let json = serde_json::to_value(&req).unwrap();
assert_eq!(json["minReplicas"], 1.0);
assert_eq!(json["maxReplicas"], 5.0);
assert_eq!(json["replicaMemoryGb"], 32.0);

// Omitted entirely when unset — mutually exclusive with the vertical
// scaling fields, so they must not serialize as null/defaults.
let json = serde_json::to_value(ServicePostRequest::default()).unwrap();
assert!(json.get("minReplicas").is_none());
assert!(json.get("maxReplicas").is_none());
assert!(json.get("replicaMemoryGb").is_none());
}

#[test]
fn deserialize_backup() {
let json = r#"{
Expand Down Expand Up @@ -813,12 +835,14 @@ fn serialize_postgres_instance_config() {
let config = PostgresInstanceConfig {
pg_config: PgConfig {
max_connections: Some(serde_json::json!(200)),
autovacuum_max_workers: Some(serde_json::json!(5)),
..Default::default()
},
pg_bouncer_config: PgBouncerConfig::default(),
};
let json = serde_json::to_value(&config).unwrap();
assert_eq!(json["pgConfig"]["max_connections"], 200);
assert_eq!(json["pgConfig"]["autovacuum_max_workers"], 5);
assert!(json.get("pgBouncerConfig").is_some());
}

Expand Down Expand Up @@ -850,6 +874,54 @@ fn serialize_postgres_instance_config_always_includes_both_nested() {
);
}

#[test]
fn serialize_clickpipe_object_storage_ingestion_controls() {
let source = ClickPipePostObjectStorageSource {
url: "https://bucket.s3.amazonaws.com/events/*.json".to_string(),
skip_initial_load: Some(true),
start_after: Some("events/2026-06-01/".to_string()),
..Default::default()
};
let json = serde_json::to_value(&source).unwrap();
assert_eq!(json["skipInitialLoad"], true);
assert_eq!(json["startAfter"], "events/2026-06-01/");

// Omitted from the wire when unset.
let json = serde_json::to_value(ClickPipePostObjectStorageSource::default()).unwrap();
assert!(json.get("skipInitialLoad").is_none());
assert!(json.get("startAfter").is_none());
}

#[test]
fn deserialize_scaling_schedule_entry_fixed_scaling_fields() {
let json = r#"{
"id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
"name": "weekday-peak",
"weekdays": [1, 2, 3, 4, 5],
"startHourUtc": 8,
"endHourUtc": 18,
"isActiveNow": false,
"numReplicas": 3,
"replicaMemoryGb": 16
}"#;
let entry: ScalingScheduleEntry = serde_json::from_str(json).unwrap();
assert_eq!(entry.num_replicas, Some(3));
assert_eq!(entry.replica_memory_gb, Some(16.0));

let req = ScalingScheduleEntryRequest {
name: entry.name.clone(),
weekdays: entry.weekdays.clone(),
start_hour_utc: entry.start_hour_utc,
end_hour_utc: entry.end_hour_utc,
num_replicas: entry.num_replicas,
replica_memory_gb: entry.replica_memory_gb,
..Default::default()
};
let json = serde_json::to_value(&req).unwrap();
assert_eq!(json["numReplicas"], 3);
assert_eq!(json["replicaMemoryGb"], 16.0);
}

#[test]
fn serialize_postgres_instance_config_default_envelope() {
// Default envelope serializes to the minimal accepted body shape.
Expand Down Expand Up @@ -1206,14 +1278,23 @@ fn deserialize_postgres_instance_config_string_wrapped_numbers() {
"pgConfig": {
"max_connections": "100",
"random_page_cost": "1.1",
"max_worker_processes": 8
"max_worker_processes": 8,
"autovacuum_naptime": "5s",
"autovacuum_vacuum_scale_factor": "0.2",
"autovacuum_max_workers": 3
},
"pgBouncerConfig": {}
}"#;
let config: PostgresInstanceConfig = serde_json::from_str(json).unwrap();
assert_eq!(config.pg_config.max_connections, Some(serde_json::json!("100")));
assert_eq!(config.pg_config.random_page_cost, Some(serde_json::json!("1.1")));
assert_eq!(config.pg_config.max_worker_processes, Some(serde_json::json!(8)));
assert_eq!(config.pg_config.autovacuum_naptime, Some(serde_json::json!("5s")));
assert_eq!(
config.pg_config.autovacuum_vacuum_scale_factor,
Some(serde_json::json!("0.2"))
);
assert_eq!(config.pg_config.autovacuum_max_workers, Some(serde_json::json!(3)));
}

#[test]
Expand Down
15 changes: 15 additions & 0 deletions crates/clickhouse-cloud-api/tests/spec_coverage_test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,13 @@ const OPTIONALITY_EXEMPTIONS: &[(&str, &str)] = &[
("ServicePostRequest", "minReplicaMemoryGb"),
("ServicePostRequest", "minTotalMemoryGb"),
("ServicePostRequest", "numReplicas"),
// Horizontal-autoscaling trio: mutually exclusive with numReplicas,
// min/maxReplicaMemoryGb, and min/maxTotalMemoryGb per the field
// descriptions, so they must be omittable despite the heuristic
// inferring required.
("ServicePostRequest", "maxReplicas"),
("ServicePostRequest", "minReplicas"),
("ServicePostRequest", "replicaMemoryGb"),
("ServicePostRequest", "privateEndpointIds"),
("ServicePostRequest", "privatePreviewTermsChecked"),
("ServicePostRequest", "profile"),
Expand Down Expand Up @@ -420,6 +427,14 @@ const OPTIONALITY_EXEMPTIONS: &[(&str, &str)] = &[
// POST bodies (omitting either yields `BAD_REQUEST: ... 'undefined'`),
// matching the spec's `required` listing. See #163 for the behaviour
// matrix evidence.
("PgConfig", "autovacuum_analyze_scale_factor"),
("PgConfig", "autovacuum_max_workers"),
("PgConfig", "autovacuum_naptime"),
("PgConfig", "autovacuum_vacuum_cost_delay"),
("PgConfig", "autovacuum_vacuum_cost_limit"),
("PgConfig", "autovacuum_vacuum_insert_scale_factor"),
("PgConfig", "autovacuum_vacuum_scale_factor"),
("PgConfig", "autovacuum_work_mem"),
("PgConfig", "default_transaction_isolation"),
("PgConfig", "effective_cache_size"),
("PgConfig", "effective_io_concurrency"),
Expand Down
6 changes: 6 additions & 0 deletions crates/clickhousectl/src/cloud/commands.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -702,6 +702,9 @@ fn build_create_service_request(
enable_core_dumps: opts.enable_core_dumps,
// Fields not exposed in CLI
byoc_id: None,
min_replicas: None,
max_replicas: None,
replica_memory_gb: None,
// Deprecated fields — only exist (and stay None) under the
// `deprecated-fields` feature; gated out of the struct otherwise.
#[cfg(feature = "deprecated-fields")]
Expand Down Expand Up @@ -1042,6 +1045,9 @@ pub async fn clickpipe_create_s3(
azure_container_name: args.azure_container_name.clone(),
path: args.path.clone(),
service_account_key,
// Not exposed as CLI flags yet
skip_initial_load: None,
start_after: None,
};

let request = ClickPipePostRequest {
Expand Down