From 1a11f884fe7a31ba1a0696973236ff90c2f5fa73 Mon Sep 17 00:00:00 2001 From: sdairs Date: Tue, 7 Jul 2026 13:11:40 +0100 Subject: [PATCH] Resolve OpenAPI drift: object storage ingestion controls, pg autovacuum, horizontal autoscaling (#287) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the 21 struct fields the live spec has grown since the last snapshot refresh, so API data for them is no longer silently dropped: - ClickPipe{,Post,Patch}ObjectStorageSource: skipInitialLoad, startAfter (nullable -> Option, no exemptions needed) - ScalingScheduleEntry{,Request}: numReplicas, replicaMemoryGb (optional via the schemas' required arrays) - ServicePostRequest: minReplicas, maxReplicas, replicaMemoryGb — the horizontal-autoscaling trio, Option to match the numReplicas sibling, with OPTIONALITY_EXEMPTIONS entries (mutually exclusive with the vertical scaling fields, so must be omittable) - PgConfig: 8 autovacuum_* fields as Option (string- or-number union like every other pgConfig setting), with exemption entries under the existing #163 partial-update rationale Refresh the vendored snapshot, update the exhaustive struct literals with behaviour-preserving defaults (CLI flag exposure is a separate follow-up), extend serde tests for all new fields, and exercise autovacuum_max_workers in the Postgres config integration round-trip. The scaling/horizontal fields are org-feature-gated and skipInitialLoad/startAfter change pipe ingestion behaviour, so those get serde coverage only. Closes #287 Co-Authored-By: Claude Fable 5 --- .../clickhouse_cloud_openapi.json | 243 ++++++++++++++++-- crates/clickhouse-cloud-api/src/models.rs | 42 +++ .../tests/common/support.rs | 2 + .../tests/integration_postgres_test.rs | 39 ++- .../tests/integration_test.rs | 4 + .../clickhouse-cloud-api/tests/models_test.rs | 83 +++++- .../tests/spec_coverage_test.rs | 15 ++ crates/clickhousectl/src/cloud/commands.rs | 6 + 8 files changed, 400 insertions(+), 34 deletions(-) diff --git a/crates/clickhouse-cloud-api/clickhouse_cloud_openapi.json b/crates/clickhouse-cloud-api/clickhouse_cloud_openapi.json index 9ce772a..c88ef34 100644 --- a/crates/clickhouse-cloud-api/clickhouse_cloud_openapi.json +++ b/crates/clickhouse-cloud-api/clickhouse_cloud_openapi.json @@ -5,7 +5,7 @@ "version": "1.0", "contact": { "name": "ClickHouse Support", - "url": "https://clickhouse.com/docs/en/cloud/manage/openapi?referrer=openapi-887333", + "url": "https://clickhouse.com/docs/en/cloud/manage/openapi?referrer=openapi-897534", "email": "support@clickhouse.com" } }, @@ -13534,6 +13534,21 @@ ], "example": "https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue" }, + "skipInitialLoad": { + "description": "If set to true, skips the initial load and only ingests files delivered by queue notifications. Only applicable when queueUrl is provided.", + "type": [ + "boolean", + "null" + ] + }, + "startAfter": { + "description": "Start continuous ingestion after this object key. Cannot be provided when skipInitialLoad is true.", + "type": [ + "string", + "null" + ], + "example": "events/2026-06-01/" + }, "authentication": { "description": "Authentication method. IAM_USER is for S3, GCS, and DigitalOcean Spaces. IAM_ROLE is for S3 only. SERVICE_ACCOUNT is for GCS only. CONNECTION_STRING is for Azure Blob Storage. PUBLIC uses no authentication.", "type": [ @@ -13656,6 +13671,21 @@ ], "example": "https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue" }, + "skipInitialLoad": { + "description": "If set to true, skips the initial load and only ingests files delivered by queue notifications. Only applicable when queueUrl is provided.", + "type": [ + "boolean", + "null" + ] + }, + "startAfter": { + "description": "Start continuous ingestion after this object key. Cannot be provided when skipInitialLoad is true.", + "type": [ + "string", + "null" + ], + "example": "events/2026-06-01/" + }, "authentication": { "description": "Authentication method. IAM_USER is for S3, GCS, and DigitalOcean Spaces. IAM_ROLE is for S3 only. SERVICE_ACCOUNT is for GCS only. CONNECTION_STRING is for Azure Blob Storage. PUBLIC uses no authentication.", "type": [ @@ -13722,6 +13752,21 @@ }, "ClickPipePatchObjectStorageSource": { "properties": { + "skipInitialLoad": { + "description": "If set to true, skips the initial load and only ingests files delivered by queue notifications. Only applicable when queueUrl is provided.", + "type": [ + "boolean", + "null" + ] + }, + "startAfter": { + "description": "Start continuous ingestion after this object key. Cannot be provided when skipInitialLoad is true.", + "type": [ + "string", + "null" + ], + "example": "events/2026-06-01/" + }, "authentication": { "description": "Authentication method. IAM_USER is for S3, GCS, and DigitalOcean Spaces. IAM_ROLE is for S3 only. SERVICE_ACCOUNT is for GCS only. CONNECTION_STRING is for Azure Blob Storage. PUBLIC uses no authentication.", "type": [ @@ -14274,7 +14319,7 @@ "example": "cdc" }, "replicationMechanism": { - "description": "MySQL replication mechanism: \"GTID\" (Global Transaction Identifier) or \"FILE_POS\" (binary log file and position). Defaults to GTID if not specified.", + "description": "MySQL replication mechanism: \"GTID\" (Global Transaction Identifier) or \"FILE_POS\" (binary log file and position). Defaults to \"GTID\" if not specified. MariaDB supports \"GTID\" only. For \"FILE_POS\" on MySQL, contact support.", "type": "string", "enum": [ "GTID", @@ -16201,19 +16246,27 @@ "maximum": 24 }, "minReplicaMemoryGb": { - "description": "Minimum memory per replica (Gb) during this window.", + "description": "Minimum memory per replica (Gb) during this window. Present only for vertical entries; horizontal entries use replicaMemoryGb instead.", "type": "number" }, "maxReplicaMemoryGb": { - "description": "Maximum memory per replica (Gb) during this window.", + "description": "Maximum memory per replica (Gb) during this window. Present only for vertical entries; horizontal entries use replicaMemoryGb instead.", "type": "number" }, + "replicaMemoryGb": { + "description": "Fixed memory per replica (Gb) during this window. Present only for horizontal entries; vertical entries use minReplicaMemoryGb/maxReplicaMemoryGb instead.", + "type": "number" + }, + "numReplicas": { + "description": "Fixed replica count for a vertical entry. Responses express the fixed count as equal minReplicas and maxReplicas rather than numReplicas.", + "type": "integer" + }, "minReplicas": { - "description": "Minimum number of replicas during this window. Must equal maxReplicas — schedule entries specify a fixed replica count per window.", + "description": "Number of replicas during this window. For a horizontal entry the replica count scales from minReplicas up to maxReplicas; for a vertical entry both equal the fixed count.", "type": "integer" }, "maxReplicas": { - "description": "Maximum number of replicas during this window. Must equal minReplicas — schedule entries specify a fixed replica count per window.", + "description": "Maximum replicas during this window for a horizontal entry; equal to the fixed count for a vertical entry.", "type": "integer" }, "idleScaling": { @@ -16326,20 +16379,42 @@ "example": 17 }, "minReplicaMemoryGb": { - "description": "Minimum memory per replica (Gb) during this window.", - "type": "number" + "description": "Minimum memory per replica (Gb) for a vertical entry. Mutually exclusive with replicaMemoryGb. The upper bound is tier-dependent (lower for non-paid organizations) and enforced when the entry is applied.", + "type": "number", + "minimum": 8, + "maximum": 356, + "multipleOf": 4 }, "maxReplicaMemoryGb": { - "description": "Maximum memory per replica (Gb) during this window.", - "type": "number" + "description": "Maximum memory per replica (Gb) for a vertical entry. Mutually exclusive with replicaMemoryGb. The upper bound is tier-dependent (lower for non-paid organizations) and enforced when the entry is applied.", + "type": "number", + "minimum": 8, + "maximum": 356, + "multipleOf": 4 + }, + "replicaMemoryGb": { + "description": "Fixed memory per replica (Gb) for a horizontal entry. A horizontal entry requires both this field and a minReplicas/maxReplicas range. Mutually exclusive with minReplicaMemoryGb/maxReplicaMemoryGb and numReplicas. Requires horizontal autoscaling to be enabled for the organization. The upper bound is tier-dependent (lower for non-paid organizations) and enforced when the entry is applied.", + "type": "number", + "minimum": 8, + "maximum": 356, + "multipleOf": 4, + "example": 16 + }, + "numReplicas": { + "description": "Fixed replica count for a vertical entry. Used for vertical autoscaling (fixed replica count, variable memory). Mutually exclusive with minReplicas/maxReplicas and replicaMemoryGb. The per-service replica maximum is variable (tier-dependent, configurable per service) and enforced when the entry is applied, not at request time.", + "type": "integer", + "minimum": 1, + "example": 3 }, "minReplicas": { - "description": "Minimum number of replicas during this window.", - "type": "integer" + "description": "Minimum number of replicas for a horizontal entry, where the replica count scales between minReplicas and maxReplicas. Must be provided together with maxReplicas. Mutually exclusive with numReplicas. The per-service replica maximum is variable (tier-dependent, configurable per service) and enforced when the entry is applied, not at request time.", + "type": "integer", + "minimum": 1 }, "maxReplicas": { - "description": "Maximum number of replicas during this window.", - "type": "integer" + "description": "Maximum number of replicas for a horizontal entry, where the replica count scales between minReplicas and maxReplicas. Must be provided together with minReplicas. Mutually exclusive with numReplicas. The per-service replica maximum is variable (tier-dependent, configurable per service) and enforced when the entry is applied, not at request time.", + "type": "integer", + "minimum": 1 }, "idleScaling": { "description": "Whether idle scaling is enabled during this window.", @@ -16589,21 +16664,21 @@ }, "numReplicas": { "description": "Number of replicas for the service. The number of replicas must be between 2 and 20 for the first service in a warehouse. Services that are created in an existing warehouse can have a number of replicas as low as 1. Further restrictions may apply based on your organization's tier. It defaults to 1 for the BASIC tier and 3 for the SCALE and ENTERPRISE tiers. Present only when the service uses vertical autoscaling. For horizontal autoscaling, use minReplicas and maxReplicas instead.", - "type": "number", + "type": "integer", "minimum": 1, "maximum": 20, "example": 3 }, "minReplicas": { "description": "Minimum number of replicas for horizontal autoscaling. Present only when the service uses horizontal autoscaling.", - "type": "number", + "type": "integer", "minimum": 1, "maximum": 20, "example": 1 }, "maxReplicas": { "description": "Maximum number of replicas for horizontal autoscaling. Present only when the service uses horizontal autoscaling.", - "type": "number", + "type": "integer", "minimum": 1, "maximum": 20, "example": 5 @@ -23923,6 +23998,102 @@ "lz4", "zstd" ] + }, + "autovacuum_max_workers": { + "type": [ + "string", + "integer" + ], + "description": "Maximum number of autovacuum worker processes that can run at the same time. Workers share a single cost-limit budget, so raising this alone may not speed up vacuuming.", + "externalDocs": { + "url": "https://postgresqlco.nf/doc/en/param/autovacuum_max_workers/" + }, + "example": 8, + "minimum": 1 + }, + "autovacuum_naptime": { + "type": [ + "string", + "integer" + ], + "description": "Minimum delay between autovacuum runs. Lower values make autovacuum check for work more frequently.", + "externalDocs": { + "url": "https://postgresqlco.nf/doc/en/param/autovacuum_naptime/" + }, + "example": "5s", + "minimum": 1 + }, + "autovacuum_work_mem": { + "type": [ + "string", + "integer" + ], + "description": "Maximum memory each autovacuum worker uses to track dead tuples. Higher values reduce repeated index-vacuum passes on large tables. Use -1 to fall back to maintenance_work_mem.", + "externalDocs": { + "url": "https://postgresqlco.nf/doc/en/param/autovacuum_work_mem/" + }, + "example": "64000kB", + "minimum": 1024 + }, + "autovacuum_vacuum_scale_factor": { + "type": [ + "string", + "number" + ], + "description": "Fraction of a table's rows that must change before autovacuum runs. Lower values vacuum large tables more frequently.", + "externalDocs": { + "url": "https://postgresqlco.nf/doc/en/param/autovacuum_vacuum_scale_factor/" + }, + "example": 0.2, + "minimum": 0 + }, + "autovacuum_analyze_scale_factor": { + "type": [ + "string", + "number" + ], + "description": "Fraction of a table's rows that must change before autovacuum runs ANALYZE to refresh planner statistics.", + "externalDocs": { + "url": "https://postgresqlco.nf/doc/en/param/autovacuum_analyze_scale_factor/" + }, + "example": 0.1, + "minimum": 0 + }, + "autovacuum_vacuum_insert_scale_factor": { + "type": [ + "string", + "number" + ], + "description": "Fraction of a table's rows that must be inserted before autovacuum runs. Helps vacuum insert-heavy, rarely-updated tables.", + "externalDocs": { + "url": "https://postgresqlco.nf/doc/en/param/autovacuum_vacuum_insert_scale_factor/" + }, + "example": 0.2, + "minimum": 0 + }, + "autovacuum_vacuum_cost_limit": { + "type": [ + "string", + "integer" + ], + "description": "Cost-accounting limit shared across all autovacuum workers before they pause. Use -1 to inherit vacuum_cost_limit.", + "externalDocs": { + "url": "https://postgresqlco.nf/doc/en/param/autovacuum_vacuum_cost_limit/" + }, + "example": "-1", + "minimum": 1 + }, + "autovacuum_vacuum_cost_delay": { + "type": [ + "string", + "integer" + ], + "description": "Time autovacuum sleeps when the cost limit is reached. Lower values speed up vacuuming at the cost of more I/O.", + "externalDocs": { + "url": "https://postgresqlco.nf/doc/en/param/autovacuum_vacuum_cost_delay/" + }, + "example": "2ms", + "minimum": 0 } }, "additionalProperties": false, @@ -25054,11 +25225,33 @@ }, "numReplicas": { "description": "Number of replicas for the service. The number of replicas must be between 2 and 20 for the first service in a warehouse. Services that are created in an existing warehouse can have a number of replicas as low as 1. Further restrictions may apply based on your organization's tier. It defaults to 1 for the BASIC tier and 3 for the SCALE and ENTERPRISE tiers. Present only when the service uses vertical autoscaling. For horizontal autoscaling, use minReplicas and maxReplicas instead.", - "type": "number", + "type": "integer", "minimum": 1, "maximum": 20, "example": 3 }, + "minReplicas": { + "description": "Minimum number of replicas for horizontal autoscaling. Must be provided together with maxReplicas and replicaMemoryGb. Mutually exclusive with numReplicas, minReplicaMemoryGb/maxReplicaMemoryGb, and minTotalMemoryGb/maxTotalMemoryGb. Requires horizontal autoscaling to be enabled for the organization.", + "type": "integer", + "minimum": 1, + "maximum": 20, + "example": 1 + }, + "maxReplicas": { + "description": "Maximum number of replicas for horizontal autoscaling. Must be provided together with minReplicas and replicaMemoryGb. Mutually exclusive with numReplicas, minReplicaMemoryGb/maxReplicaMemoryGb, and minTotalMemoryGb/maxTotalMemoryGb. Requires horizontal autoscaling to be enabled for the organization.", + "type": "integer", + "minimum": 1, + "maximum": 20, + "example": 5 + }, + "replicaMemoryGb": { + "description": "Fixed memory per replica in Gb for horizontal autoscaling. Must be provided together with minReplicas/maxReplicas. Must be a multiple of 4, at least 8 Gb, and at most 120 Gb for non paid services or 356 Gb for paid services. Mutually exclusive with minReplicaMemoryGb/maxReplicaMemoryGb and minTotalMemoryGb/maxTotalMemoryGb. Requires horizontal autoscaling to be enabled for the organization.", + "type": "number", + "minimum": 8, + "maximum": 356, + "multipleOf": 4, + "example": 32 + }, "idleScaling": { "description": "When set to true the service is allowed to scale down to zero when idle. True by default.", "type": "boolean" @@ -25236,7 +25429,7 @@ }, "numReplicas": { "description": "Number of replicas for the service. The number of replicas must be between 2 and 20 for the first service in a warehouse. Services that are created in an existing warehouse can have a number of replicas as low as 1. Further restrictions may apply based on your organization's tier. It defaults to 1 for the BASIC tier and 3 for the SCALE and ENTERPRISE tiers.", - "type": "number", + "type": "integer", "minimum": 1, "maximum": 20, "example": 3 @@ -25387,21 +25580,21 @@ }, "numReplicas": { "description": "Number of replicas for the service. The number of replicas must be between 2 and 20 for the first service in a warehouse. Services that are created in an existing warehouse can have a number of replicas as low as 1. Further restrictions may apply based on your organization's tier. It defaults to 1 for the BASIC tier and 3 for the SCALE and ENTERPRISE tiers. Present only when the service uses vertical autoscaling. For horizontal autoscaling, use minReplicas and maxReplicas instead.", - "type": "number", + "type": "integer", "minimum": 1, "maximum": 20, "example": 3 }, "minReplicas": { "description": "Minimum number of replicas for horizontal autoscaling. Present only when the service uses horizontal autoscaling.", - "type": "number", + "type": "integer", "minimum": 1, "maximum": 20, "example": 1 }, "maxReplicas": { "description": "Maximum number of replicas for horizontal autoscaling. Present only when the service uses horizontal autoscaling.", - "type": "number", + "type": "integer", "minimum": 1, "maximum": 20, "example": 5 @@ -25560,21 +25753,21 @@ }, "numReplicas": { "description": "Fixed replica count for vertical autoscaling. Mutually exclusive with minReplicas/maxReplicas. Please contact support to enable adjustment of numReplicas. When switching from horizontal autoscaling mode the existing replica memory is preserved as the new vertical min/max range; include minReplicaMemoryGb/maxReplicaMemoryGb in the same request to set an explicit range.", - "type": "number", + "type": "integer", "minimum": 1, "maximum": 20, "example": 3 }, "minReplicas": { "description": "Minimum number of replicas for horizontal autoscaling. Must be provided together with maxReplicas. Mutually exclusive with numReplicas. Requires horizontal autoscaling to be enabled for the service.", - "type": "number", + "type": "integer", "minimum": 1, "maximum": 20, "example": 1 }, "maxReplicas": { "description": "Maximum number of replicas for horizontal autoscaling. Must be provided together with minReplicas. Mutually exclusive with numReplicas. Requires horizontal autoscaling to be enabled for the service.", - "type": "number", + "type": "integer", "minimum": 1, "maximum": 20, "example": 5 diff --git a/crates/clickhouse-cloud-api/src/models.rs b/crates/clickhouse-cloud-api/src/models.rs index d4a073e..5283255 100644 --- a/crates/clickhouse-cloud-api/src/models.rs +++ b/crates/clickhouse-cloud-api/src/models.rs @@ -8391,6 +8391,10 @@ pub struct ClickPipeObjectStorageSource { pub path: Option, #[serde(rename = "queueUrl", skip_serializing_if = "Option::is_none", default)] pub queue_url: Option, + #[serde(rename = "skipInitialLoad", skip_serializing_if = "Option::is_none", default)] + pub skip_initial_load: Option, + #[serde(rename = "startAfter", skip_serializing_if = "Option::is_none", default)] + pub start_after: Option, #[serde(default)] pub r#type: ClickPipeObjectStorageSourceType, #[serde(default)] @@ -8546,6 +8550,10 @@ pub struct ClickPipePatchObjectStorageSource { pub path: Option, #[serde(rename = "serviceAccountKey", skip_serializing_if = "Option::is_none", default)] pub service_account_key: Option, + #[serde(rename = "skipInitialLoad", skip_serializing_if = "Option::is_none", default)] + pub skip_initial_load: Option, + #[serde(rename = "startAfter", skip_serializing_if = "Option::is_none", default)] + pub start_after: Option, } /// `ClickPipePatchPostgresPipeRemoveTableMapping` from the ClickHouse Cloud API. @@ -8726,6 +8734,10 @@ pub struct ClickPipePostObjectStorageSource { pub queue_url: Option, #[serde(rename = "serviceAccountKey", skip_serializing_if = "Option::is_none", default)] pub service_account_key: Option, + #[serde(rename = "skipInitialLoad", skip_serializing_if = "Option::is_none", default)] + pub skip_initial_load: Option, + #[serde(rename = "startAfter", skip_serializing_if = "Option::is_none", default)] + pub start_after: Option, #[serde(default)] pub r#type: ClickPipePostObjectStorageSourceType, #[serde(default)] @@ -10981,6 +10993,10 @@ pub struct ScalingScheduleEntry { pub min_replicas: Option, #[serde(default)] pub name: String, + #[serde(rename = "numReplicas", skip_serializing_if = "Option::is_none", default)] + pub num_replicas: Option, + #[serde(rename = "replicaMemoryGb", skip_serializing_if = "Option::is_none", default)] + pub replica_memory_gb: Option, #[serde(rename = "startHourUtc", default)] pub start_hour_utc: i64, #[serde(default)] @@ -11006,6 +11022,10 @@ pub struct ScalingScheduleEntryRequest { pub min_replicas: Option, #[serde(default)] pub name: String, + #[serde(rename = "numReplicas", skip_serializing_if = "Option::is_none", default)] + pub num_replicas: Option, + #[serde(rename = "replicaMemoryGb", skip_serializing_if = "Option::is_none", default)] + pub replica_memory_gb: Option, #[serde(rename = "startHourUtc", default)] pub start_hour_utc: i64, #[serde(default)] @@ -11873,11 +11893,15 @@ pub struct ServicePostRequest { pub is_readonly: Option, #[serde(rename = "maxReplicaMemoryGb", skip_serializing_if = "Option::is_none", default)] pub max_replica_memory_gb: Option, + #[serde(rename = "maxReplicas", skip_serializing_if = "Option::is_none", default)] + pub max_replicas: Option, #[cfg(feature = "deprecated-fields")] #[serde(rename = "maxTotalMemoryGb", skip_serializing_if = "Option::is_none", default)] pub max_total_memory_gb: Option, #[serde(rename = "minReplicaMemoryGb", skip_serializing_if = "Option::is_none", default)] pub min_replica_memory_gb: Option, + #[serde(rename = "minReplicas", skip_serializing_if = "Option::is_none", default)] + pub min_replicas: Option, #[cfg(feature = "deprecated-fields")] #[serde(rename = "minTotalMemoryGb", skip_serializing_if = "Option::is_none", default)] pub min_total_memory_gb: Option, @@ -11898,6 +11922,8 @@ pub struct ServicePostRequest { pub region: ServicePostRequestRegion, #[serde(rename = "releaseChannel", skip_serializing_if = "Option::is_none", default)] pub release_channel: Option, + #[serde(rename = "replicaMemoryGb", skip_serializing_if = "Option::is_none", default)] + pub replica_memory_gb: Option, #[serde(skip_serializing_if = "Option::is_none", default)] pub tags: Option>, #[cfg(feature = "deprecated-fields")] @@ -12150,6 +12176,22 @@ pub struct PgBouncerConfig { /// `pgConfig` from the ClickHouse Cloud API. #[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] pub struct PgConfig { + #[serde(skip_serializing_if = "Option::is_none", default)] + pub autovacuum_analyze_scale_factor: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub autovacuum_max_workers: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub autovacuum_naptime: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub autovacuum_vacuum_cost_delay: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub autovacuum_vacuum_cost_limit: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub autovacuum_vacuum_insert_scale_factor: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub autovacuum_vacuum_scale_factor: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub autovacuum_work_mem: Option, #[serde(skip_serializing_if = "Option::is_none", default)] pub default_transaction_isolation: Option, #[serde(skip_serializing_if = "Option::is_none", default)] diff --git a/crates/clickhouse-cloud-api/tests/common/support.rs b/crates/clickhouse-cloud-api/tests/common/support.rs index a41633a..5a31624 100644 --- a/crates/clickhouse-cloud-api/tests/common/support.rs +++ b/crates/clickhouse-cloud-api/tests/common/support.rs @@ -914,6 +914,8 @@ pub fn scaling_schedule_entry_to_request( min_replica_memory_gb: entry.min_replica_memory_gb, min_replicas: entry.min_replicas, name: entry.name.clone(), + num_replicas: entry.num_replicas, + replica_memory_gb: entry.replica_memory_gb, start_hour_utc: entry.start_hour_utc, weekdays: entry.weekdays.clone(), } diff --git a/crates/clickhouse-cloud-api/tests/integration_postgres_test.rs b/crates/clickhouse-cloud-api/tests/integration_postgres_test.rs index a09570d..19db4c0 100644 --- a/crates/clickhouse-cloud-api/tests/integration_postgres_test.rs +++ b/crates/clickhouse-cloud-api/tests/integration_postgres_test.rs @@ -230,8 +230,9 @@ async fn cloud_postgres_crud_lifecycle() -> TestResult<()> { .await?; } - // Round-trip a single pgConfig field: PATCH max_connections to a - // new value, poll-until GET reflects it, then PATCH back. GET + // Round-trip pgConfig fields: PATCH max_connections and + // autovacuum_max_workers to new values, poll-until GET reflects + // them, then PATCH back. GET // returns numeric pgConfig values wrapped in JSON strings (the spec // types them string-or-number), so extract via pg_config_value_as_i64 // and compare tolerantly. @@ -243,12 +244,21 @@ async fn cloud_postgres_crud_lifecycle() -> TestResult<()> { .and_then(pg_config_value_as_i64) .unwrap_or(100); let target = baseline_max + 7; + // Postgres defaults autovacuum_max_workers to 3; the per-run + // service is deleted at the end, so drift on reset is harmless. + let baseline_autovacuum = baseline + .pg_config + .autovacuum_max_workers + .as_ref() + .and_then(pg_config_value_as_i64) + .unwrap_or(3); + let autovacuum_target = baseline_autovacuum + 1; failures .run( &ctx, StepKind::NonBlocking, - "patch pgConfig.max_connections", + "patch pgConfig.max_connections/autovacuum_max_workers", || { let client = client.clone(); let org_id = ctx.org_id.clone(); @@ -257,6 +267,9 @@ async fn cloud_postgres_crud_lifecycle() -> TestResult<()> { let body = PostgresInstanceConfig { pg_config: PgConfig { max_connections: Some(serde_json::json!(target)), + autovacuum_max_workers: Some(serde_json::json!( + autovacuum_target + )), ..Default::default() }, pg_bouncer_config: PgBouncerConfig::default(), @@ -283,7 +296,7 @@ async fn cloud_postgres_crud_lifecycle() -> TestResult<()> { let interval = ctx.poll_interval; async move { poll_until( - "pg_config.max_connections == target", + "pg_config.max_connections/autovacuum_max_workers == targets", timeout, interval, || { @@ -294,12 +307,19 @@ async fn cloud_postgres_crud_lifecycle() -> TestResult<()> { let resp = client .postgres_instance_config_get(&org_id, &postgres_id) .await?; - let observed = resp - .result - .and_then(|r| r.pg_config.max_connections) + let pg_config = + resp.result.map(|r| r.pg_config).unwrap_or_default(); + let observed = pg_config + .max_connections .as_ref() .and_then(pg_config_value_as_i64); - if observed == Some(target) { + let observed_autovacuum = pg_config + .autovacuum_max_workers + .as_ref() + .and_then(pg_config_value_as_i64); + if observed == Some(target) + && observed_autovacuum == Some(autovacuum_target) + { Ok(Some(())) } else { Ok(None) @@ -326,6 +346,9 @@ async fn cloud_postgres_crud_lifecycle() -> TestResult<()> { let body = PostgresInstanceConfig { pg_config: PgConfig { max_connections: Some(serde_json::json!(baseline_max)), + autovacuum_max_workers: Some(serde_json::json!( + baseline_autovacuum + )), ..Default::default() }, pg_bouncer_config: PgBouncerConfig::default(), diff --git a/crates/clickhouse-cloud-api/tests/integration_test.rs b/crates/clickhouse-cloud-api/tests/integration_test.rs index 4323736..711b526 100644 --- a/crates/clickhouse-cloud-api/tests/integration_test.rs +++ b/crates/clickhouse-cloud-api/tests/integration_test.rs @@ -2032,6 +2032,10 @@ async fn cloud_service_crud_lifecycle() -> TestResult<()> { max_replicas: Some(base_replicas as i64), idle_scaling: Some(true), idle_timeout_minutes: Some(5), + // Vertical entry expressed as equal min/max; the fixed-count + // and horizontal-autoscaling forms need org-level enablement. + num_replicas: None, + replica_memory_gb: None, }; let upserted = failures diff --git a/crates/clickhouse-cloud-api/tests/models_test.rs b/crates/clickhouse-cloud-api/tests/models_test.rs index 0e5f35c..ff29f2b 100644 --- a/crates/clickhouse-cloud-api/tests/models_test.rs +++ b/crates/clickhouse-cloud-api/tests/models_test.rs @@ -151,6 +151,28 @@ fn serialize_service_post_request() { assert_eq!(json["ipAccessList"][0]["source"], "0.0.0.0/0"); } +#[test] +fn serialize_service_post_request_horizontal_autoscaling() { + let req = ServicePostRequest { + name: "horizontal-service".to_string(), + min_replicas: Some(1.0), + max_replicas: Some(5.0), + replica_memory_gb: Some(32.0), + ..Default::default() + }; + let json = serde_json::to_value(&req).unwrap(); + assert_eq!(json["minReplicas"], 1.0); + assert_eq!(json["maxReplicas"], 5.0); + assert_eq!(json["replicaMemoryGb"], 32.0); + + // Omitted entirely when unset — mutually exclusive with the vertical + // scaling fields, so they must not serialize as null/defaults. + let json = serde_json::to_value(ServicePostRequest::default()).unwrap(); + assert!(json.get("minReplicas").is_none()); + assert!(json.get("maxReplicas").is_none()); + assert!(json.get("replicaMemoryGb").is_none()); +} + #[test] fn deserialize_backup() { let json = r#"{ @@ -813,12 +835,14 @@ fn serialize_postgres_instance_config() { let config = PostgresInstanceConfig { pg_config: PgConfig { max_connections: Some(serde_json::json!(200)), + autovacuum_max_workers: Some(serde_json::json!(5)), ..Default::default() }, pg_bouncer_config: PgBouncerConfig::default(), }; let json = serde_json::to_value(&config).unwrap(); assert_eq!(json["pgConfig"]["max_connections"], 200); + assert_eq!(json["pgConfig"]["autovacuum_max_workers"], 5); assert!(json.get("pgBouncerConfig").is_some()); } @@ -850,6 +874,54 @@ fn serialize_postgres_instance_config_always_includes_both_nested() { ); } +#[test] +fn serialize_clickpipe_object_storage_ingestion_controls() { + let source = ClickPipePostObjectStorageSource { + url: "https://bucket.s3.amazonaws.com/events/*.json".to_string(), + skip_initial_load: Some(true), + start_after: Some("events/2026-06-01/".to_string()), + ..Default::default() + }; + let json = serde_json::to_value(&source).unwrap(); + assert_eq!(json["skipInitialLoad"], true); + assert_eq!(json["startAfter"], "events/2026-06-01/"); + + // Omitted from the wire when unset. + let json = serde_json::to_value(ClickPipePostObjectStorageSource::default()).unwrap(); + assert!(json.get("skipInitialLoad").is_none()); + assert!(json.get("startAfter").is_none()); +} + +#[test] +fn deserialize_scaling_schedule_entry_fixed_scaling_fields() { + let json = r#"{ + "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "name": "weekday-peak", + "weekdays": [1, 2, 3, 4, 5], + "startHourUtc": 8, + "endHourUtc": 18, + "isActiveNow": false, + "numReplicas": 3, + "replicaMemoryGb": 16 + }"#; + let entry: ScalingScheduleEntry = serde_json::from_str(json).unwrap(); + assert_eq!(entry.num_replicas, Some(3)); + assert_eq!(entry.replica_memory_gb, Some(16.0)); + + let req = ScalingScheduleEntryRequest { + name: entry.name.clone(), + weekdays: entry.weekdays.clone(), + start_hour_utc: entry.start_hour_utc, + end_hour_utc: entry.end_hour_utc, + num_replicas: entry.num_replicas, + replica_memory_gb: entry.replica_memory_gb, + ..Default::default() + }; + let json = serde_json::to_value(&req).unwrap(); + assert_eq!(json["numReplicas"], 3); + assert_eq!(json["replicaMemoryGb"], 16.0); +} + #[test] fn serialize_postgres_instance_config_default_envelope() { // Default envelope serializes to the minimal accepted body shape. @@ -1206,7 +1278,10 @@ fn deserialize_postgres_instance_config_string_wrapped_numbers() { "pgConfig": { "max_connections": "100", "random_page_cost": "1.1", - "max_worker_processes": 8 + "max_worker_processes": 8, + "autovacuum_naptime": "5s", + "autovacuum_vacuum_scale_factor": "0.2", + "autovacuum_max_workers": 3 }, "pgBouncerConfig": {} }"#; @@ -1214,6 +1289,12 @@ fn deserialize_postgres_instance_config_string_wrapped_numbers() { assert_eq!(config.pg_config.max_connections, Some(serde_json::json!("100"))); assert_eq!(config.pg_config.random_page_cost, Some(serde_json::json!("1.1"))); assert_eq!(config.pg_config.max_worker_processes, Some(serde_json::json!(8))); + assert_eq!(config.pg_config.autovacuum_naptime, Some(serde_json::json!("5s"))); + assert_eq!( + config.pg_config.autovacuum_vacuum_scale_factor, + Some(serde_json::json!("0.2")) + ); + assert_eq!(config.pg_config.autovacuum_max_workers, Some(serde_json::json!(3))); } #[test] diff --git a/crates/clickhouse-cloud-api/tests/spec_coverage_test.rs b/crates/clickhouse-cloud-api/tests/spec_coverage_test.rs index deee406..c9211f4 100644 --- a/crates/clickhouse-cloud-api/tests/spec_coverage_test.rs +++ b/crates/clickhouse-cloud-api/tests/spec_coverage_test.rs @@ -323,6 +323,13 @@ const OPTIONALITY_EXEMPTIONS: &[(&str, &str)] = &[ ("ServicePostRequest", "minReplicaMemoryGb"), ("ServicePostRequest", "minTotalMemoryGb"), ("ServicePostRequest", "numReplicas"), + // Horizontal-autoscaling trio: mutually exclusive with numReplicas, + // min/maxReplicaMemoryGb, and min/maxTotalMemoryGb per the field + // descriptions, so they must be omittable despite the heuristic + // inferring required. + ("ServicePostRequest", "maxReplicas"), + ("ServicePostRequest", "minReplicas"), + ("ServicePostRequest", "replicaMemoryGb"), ("ServicePostRequest", "privateEndpointIds"), ("ServicePostRequest", "privatePreviewTermsChecked"), ("ServicePostRequest", "profile"), @@ -420,6 +427,14 @@ const OPTIONALITY_EXEMPTIONS: &[(&str, &str)] = &[ // POST bodies (omitting either yields `BAD_REQUEST: ... 'undefined'`), // matching the spec's `required` listing. See #163 for the behaviour // matrix evidence. + ("PgConfig", "autovacuum_analyze_scale_factor"), + ("PgConfig", "autovacuum_max_workers"), + ("PgConfig", "autovacuum_naptime"), + ("PgConfig", "autovacuum_vacuum_cost_delay"), + ("PgConfig", "autovacuum_vacuum_cost_limit"), + ("PgConfig", "autovacuum_vacuum_insert_scale_factor"), + ("PgConfig", "autovacuum_vacuum_scale_factor"), + ("PgConfig", "autovacuum_work_mem"), ("PgConfig", "default_transaction_isolation"), ("PgConfig", "effective_cache_size"), ("PgConfig", "effective_io_concurrency"), diff --git a/crates/clickhousectl/src/cloud/commands.rs b/crates/clickhousectl/src/cloud/commands.rs index 02bbe8f..d747e5a 100644 --- a/crates/clickhousectl/src/cloud/commands.rs +++ b/crates/clickhousectl/src/cloud/commands.rs @@ -702,6 +702,9 @@ fn build_create_service_request( enable_core_dumps: opts.enable_core_dumps, // Fields not exposed in CLI byoc_id: None, + min_replicas: None, + max_replicas: None, + replica_memory_gb: None, // Deprecated fields — only exist (and stay None) under the // `deprecated-fields` feature; gated out of the struct otherwise. #[cfg(feature = "deprecated-fields")] @@ -1042,6 +1045,9 @@ pub async fn clickpipe_create_s3( azure_container_name: args.azure_container_name.clone(), path: args.path.clone(), service_account_key, + // Not exposed as CLI flags yet + skip_initial_load: None, + start_after: None, }; let request = ClickPipePostRequest {