Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion docs/resources/warehouse.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,14 @@ warehouse = Warehouse(
tags={"env": "test"},
)
```
An adaptive warehouse sets max_query_performance_level instead of warehouse_size and cluster/scaling properties:
```python
adaptive_warehouse = Warehouse(
name="some_adaptive_warehouse",
warehouse_type="ADAPTIVE",
max_query_performance_level="LARGE",
)
```


### YAML
Expand Down Expand Up @@ -66,16 +74,24 @@ warehouses:
tags:
env: test
```
An adaptive warehouse in yaml:
```yaml
warehouses:
- name: some_adaptive_warehouse
warehouse_type: ADAPTIVE
max_query_performance_level: LARGE
```


## Fields

* `name` (string, required) - The name of the warehouse.
* `owner` (string) - The owner of the warehouse. Defaults to "SYSADMIN".
* `warehouse_type` (string or [WarehouseType](warehouse_type.md)) - The type of the warehouse, either STANDARD or SNOWPARK-OPTIMIZED. Defaults to STANDARD.
* `warehouse_type` (string or [WarehouseType](warehouse_type.md)) - The type of the warehouse: STANDARD, SNOWPARK-OPTIMIZED, or ADAPTIVE. Defaults to STANDARD. ADAPTIVE warehouses do not support warehouse_size, min_cluster_count, max_cluster_count, scaling_policy, auto_suspend, auto_resume, initially_suspended, enable_query_acceleration, query_acceleration_max_scale_factor, resource_constraint, or generation.
* `warehouse_size` (string or [WarehouseSize](warehouse_size.md)) - The size of the warehouse which defines the compute and storage capacity.
* `generation` (string or [WarehouseGeneration](warehouse_generation.md)) - The standard warehouse generation, either "1" or "2".
* `resource_constraint` (string or [WarehouseResourceConstraint](warehouse_resource_constraint.md)) - The warehouse resource constraint, either STANDARD_GEN_1/2 for standard warehouses or MEMORY_* for Snowpark-optimized warehouses.
* `max_query_performance_level` (string or [WarehouseSize](warehouse_size.md)) - The maximum size an ADAPTIVE warehouse may scale to: XSMALL, SMALL, MEDIUM, LARGE, XLARGE, XXLARGE, XXXLARGE, or X4LARGE. Only valid for ADAPTIVE warehouses; Snowflake defaults to XLARGE if omitted.
* `max_cluster_count` (int) - The maximum number of clusters for the warehouse.
* `min_cluster_count` (int) - The minimum number of clusters for the warehouse.
* `scaling_policy` (string or [WarehouseScalingPolicy](warehouse_scaling_policy.md)) - The policy that defines how the warehouse scales.
Expand All @@ -90,3 +106,5 @@ warehouses:
* `statement_queued_timeout_in_seconds` (int) - The time in seconds a statement can be queued before it times out.
* `statement_timeout_in_seconds` (int) - The time in seconds a statement can run before it times out.
* `tags` (dict) - Tags for the warehouse.


16 changes: 14 additions & 2 deletions snowcap/blueprint.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
)
from .exceptions import (
DuplicateResourceException,
InvalidResourceException,
MissingPrivilegeException,
MissingResourceException,
NonConformingPlanException,
Expand Down Expand Up @@ -744,8 +745,8 @@ def _get_key_properties(change: "ResourceChange", resource_type: ResourceType) -
if not is_grant and "owner" in data and data["owner"]:
props.append(f"owner: {data['owner']}")

# Show size for warehouses
if "warehouse_size" in data:
# Show size for warehouses (adaptive warehouses have no size, so warehouse_size is None)
if data.get("warehouse_size") is not None:
props.append(f"size: {data['warehouse_size']}")

if props:
Expand Down Expand Up @@ -2312,6 +2313,17 @@ def _diff_resource_data(lhs: dict, rhs: dict) -> dict:

delta = {k: v for k, v in delta.items() if k not in ignore_fields}
if delta:
# Snowflake doesn't support converting to or from X5LARGE/X6LARGE warehouses
# (https://docs.snowflake.com/en/user-guide/warehouses-adaptive), so fail at
# plan time instead of erroring mid-apply.
if urn.resource_type == ResourceType.WAREHOUSE and "warehouse_type" in delta:
converting_adaptive = "ADAPTIVE" in (delta["warehouse_type"], remote_state[urn].get("warehouse_type"))
sizes = {remote_state[urn].get("warehouse_size"), manifest_item.data.get("warehouse_size")}
if converting_adaptive and sizes & {"X5LARGE", "X6LARGE"}:
raise InvalidResourceException(
f"{urn}: Snowflake does not support converting an X5LARGE or X6LARGE warehouse "
"to ADAPTIVE, or converting an ADAPTIVE warehouse to those sizes"
)
changes.append(
UpdateResource(
urn,
Expand Down
23 changes: 22 additions & 1 deletion snowcap/data_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
attribute_is_resource_name,
resource_name_from_snowflake_metadata,
)
from .resources.warehouse import ADAPTIVE_UNSUPPORTED_FIELDS

__this__ = sys.modules[__name__]

Expand Down Expand Up @@ -3166,6 +3167,9 @@ def fetch_warehouse(session: SnowflakeConnection, fqn: FQN, include_params: bool
if generation is not None:
generation = str(generation)
resource_constraint = _normalize_snowflake_optional(data.get("resource_constraint"), upper=True)
max_query_performance_level = _normalize_snowflake_optional(
data.get("max_query_performance_level"), upper=True
)

if warehouse_type == "STANDARD":
if resource_constraint is None and generation in {"1", "2"}:
Expand All @@ -3178,13 +3182,22 @@ def fetch_warehouse(session: SnowflakeConnection, fqn: FQN, include_params: bool
if query_accel is not None:
query_accel = str(query_accel).lower() == "true"

# Adaptive warehouses may report ''/'null'/'ADAPTIVE' in the size column instead of a
# WarehouseSize value; ADAPTIVE_UNSUPPORTED_FIELDS nulls warehouse_size out below regardless,
# so this conversion just needs to not crash on those values.
try:
warehouse_size = str(WarehouseSize(data["size"]))
except ValueError:
warehouse_size = None

warehouse_dict = {
"name": _quote_snowflake_identifier(data["name"]),
"owner": _get_owner_identifier(data),
"warehouse_type": warehouse_type,
"warehouse_size": str(WarehouseSize(data["size"])),
"warehouse_size": warehouse_size,
"generation": generation,
"resource_constraint": resource_constraint,
"max_query_performance_level": max_query_performance_level,
"auto_suspend": data["auto_suspend"],
"auto_resume": data["auto_resume"] == "true",
"comment": data["comment"] or None,
Expand All @@ -3201,6 +3214,14 @@ def fetch_warehouse(session: SnowflakeConnection, fqn: FQN, include_params: bool
"statement_timeout_in_seconds": statement_timeout,
}

# ADAPTIVE warehouses don't support these properties (Snowflake computes them
# automatically); null them out here using the same field set __post_init__ validates
# against, so fetched state matches a declared ADAPTIVE spec with no spurious drift.
if warehouse_type == "ADAPTIVE":
for field_name in ADAPTIVE_UNSUPPORTED_FIELDS:
if field_name in warehouse_dict:
warehouse_dict[field_name] = None

return warehouse_dict


Expand Down
62 changes: 60 additions & 2 deletions snowcap/resources/warehouse.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from dataclasses import dataclass, field
from dataclasses import dataclass, field, fields
from typing import Optional, Union

from ..enums import AccountEdition, ParseableEnum, ResourceType, WarehouseSize
Expand All @@ -22,6 +22,7 @@
class WarehouseType(ParseableEnum):
STANDARD = "STANDARD"
SNOWPARK_OPTIMIZED = "SNOWPARK-OPTIMIZED"
ADAPTIVE = "ADAPTIVE"


class WarehouseScalingPolicy(ParseableEnum):
Expand Down Expand Up @@ -69,6 +70,23 @@ class WarehouseResourceConstraint(ParseableEnum):
WarehouseGeneration.GEN2: WarehouseResourceConstraint.STANDARD_GEN_2,
}

# Properties that don't apply to ADAPTIVE warehouses (Snowflake computes them automatically).
# Names only: __post_init__ reads each field's real default from dataclasses.fields at
# validation time rather than hand-copying defaults here. "generation" is deliberately
# excluded — the existing generation-vs-STANDARD check already rejects it for ADAPTIVE.
ADAPTIVE_UNSUPPORTED_FIELDS = (
"warehouse_size",
"min_cluster_count",
"max_cluster_count",
"scaling_policy",
"auto_suspend",
"auto_resume",
"initially_suspended",
"enable_query_acceleration",
"query_acceleration_max_scale_factor",
"resource_constraint",
)


@dataclass(unsafe_hash=True)
class _Warehouse(ResourceSpec):
Expand All @@ -78,6 +96,7 @@ class _Warehouse(ResourceSpec):
warehouse_size: WarehouseSize = WarehouseSize.XSMALL
generation: Optional[WarehouseGeneration] = None
resource_constraint: Optional[WarehouseResourceConstraint] = None
max_query_performance_level: Optional[WarehouseSize] = None
max_cluster_count: int = field(
default=1,
metadata={"edition": {AccountEdition.ENTERPRISE, AccountEdition.BUSINESS_CRITICAL}},
Expand Down Expand Up @@ -133,6 +152,24 @@ def __post_init__(self):
if uses_standard_gen2 and self.warehouse_size in {WarehouseSize.X5LARGE, WarehouseSize.X6LARGE}:
raise ValueError("Gen2 standard warehouses do not support X5LARGE or X6LARGE sizes")

if self.warehouse_type == WarehouseType.ADAPTIVE:
for field_name in ADAPTIVE_UNSUPPORTED_FIELDS:
if getattr(self, field_name) != _ADAPTIVE_FIELD_DEFAULTS[field_name]:
raise ValueError(f"{field_name} does not apply to ADAPTIVE warehouses")
setattr(self, field_name, None)

if self.max_query_performance_level is not None:
if self.warehouse_type != WarehouseType.ADAPTIVE:
raise ValueError("max_query_performance_level is only valid for ADAPTIVE warehouses")
if self.max_query_performance_level in {WarehouseSize.X5LARGE, WarehouseSize.X6LARGE}:
raise ValueError("max_query_performance_level supports XSMALL through X4LARGE")


# Real dataclass defaults for the adaptive-inapplicable fields, computed once from
# dataclasses.fields (the single source of truth) instead of rebuilding this dict on
# every ADAPTIVE warehouse construction.
_ADAPTIVE_FIELD_DEFAULTS = {f.name: f.default for f in fields(_Warehouse) if f.name in ADAPTIVE_UNSUPPORTED_FIELDS}


class Warehouse(NamedResource, TaggableResource, Resource):
"""
Expand All @@ -145,10 +182,11 @@ class Warehouse(NamedResource, TaggableResource, Resource):
Fields:
name (string, required): The name of the warehouse.
owner (string): The owner of the warehouse. Defaults to "SYSADMIN".
warehouse_type (string or WarehouseType): The type of the warehouse, either STANDARD or SNOWPARK-OPTIMIZED. Defaults to STANDARD.
warehouse_type (string or WarehouseType): The type of the warehouse: STANDARD, SNOWPARK-OPTIMIZED, or ADAPTIVE. Defaults to STANDARD. ADAPTIVE warehouses do not support warehouse_size, min_cluster_count, max_cluster_count, scaling_policy, auto_suspend, auto_resume, initially_suspended, enable_query_acceleration, query_acceleration_max_scale_factor, resource_constraint, or generation.
warehouse_size (string or WarehouseSize): The size of the warehouse which defines the compute and storage capacity.
generation (string or WarehouseGeneration): The standard warehouse generation, either "1" or "2".
resource_constraint (string or WarehouseResourceConstraint): The warehouse resource constraint, either STANDARD_GEN_1/2 for standard warehouses or MEMORY_* for Snowpark-optimized warehouses.
max_query_performance_level (string or WarehouseSize): The maximum size an ADAPTIVE warehouse may scale to: XSMALL, SMALL, MEDIUM, LARGE, XLARGE, XXLARGE, XXXLARGE, or X4LARGE. Only valid for ADAPTIVE warehouses; Snowflake defaults to XLARGE if omitted.
max_cluster_count (int): The maximum number of clusters for the warehouse.
min_cluster_count (int): The minimum number of clusters for the warehouse.
scaling_policy (string or WarehouseScalingPolicy): The policy that defines how the warehouse scales.
Expand Down Expand Up @@ -191,6 +229,15 @@ class Warehouse(NamedResource, TaggableResource, Resource):
)
```

An adaptive warehouse sets max_query_performance_level instead of warehouse_size and cluster/scaling properties:
```python
adaptive_warehouse = Warehouse(
name="some_adaptive_warehouse",
warehouse_type="ADAPTIVE",
max_query_performance_level="LARGE",
)
```

Yaml:

```yaml
Expand All @@ -217,6 +264,14 @@ class Warehouse(NamedResource, TaggableResource, Resource):
tags:
env: test
```

An adaptive warehouse in yaml:
```yaml
warehouses:
- name: some_adaptive_warehouse
warehouse_type: ADAPTIVE
max_query_performance_level: LARGE
```
"""

resource_type = ResourceType.WAREHOUSE
Expand All @@ -226,6 +281,7 @@ class Warehouse(NamedResource, TaggableResource, Resource):
warehouse_size=EnumProp("warehouse_size", WarehouseSize),
generation=EnumProp("GENERATION", WarehouseGeneration, quoted=True),
resource_constraint=EnumProp("RESOURCE_CONSTRAINT", WarehouseResourceConstraint),
max_query_performance_level=EnumProp("MAX_QUERY_PERFORMANCE_LEVEL", WarehouseSize),
max_cluster_count=IntProp("max_cluster_count"),
min_cluster_count=IntProp("min_cluster_count"),
scaling_policy=EnumProp("scaling_policy", WarehouseScalingPolicy),
Expand All @@ -252,6 +308,7 @@ def __init__(
warehouse_size: str = "XSMALL",
generation: str = None,
resource_constraint: str = None,
max_query_performance_level: str = None,
max_cluster_count: int = 1,
min_cluster_count: int = 1,
scaling_policy: str = "STANDARD",
Expand All @@ -276,6 +333,7 @@ def __init__(
warehouse_size=warehouse_size,
generation=generation,
resource_constraint=resource_constraint,
max_query_performance_level=max_query_performance_level,
max_cluster_count=max_cluster_count,
min_cluster_count=min_cluster_count,
scaling_policy=scaling_policy,
Expand Down
1 change: 1 addition & 0 deletions tests/fixtures/json/warehouse.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"warehouse_size": "XSMALL",
"generation": "2",
"resource_constraint": "STANDARD_GEN_2",
"max_query_performance_level": null,
"max_cluster_count": 1,
"min_cluster_count": 1,
"scaling_policy": "STANDARD",
Expand Down
8 changes: 8 additions & 0 deletions tests/fixtures/sql/warehouse.sql
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,11 @@ scaling_policy = economy
initially_suspended = true
;

CREATE WAREHOUSE ADAPTIVE_WH
WITH
WAREHOUSE_TYPE = 'ADAPTIVE'
MAX_QUERY_PERFORMANCE_LEVEL = LARGE
STATEMENT_TIMEOUT_IN_SECONDS = 3600
COMMENT = 'My adaptive warehouse'
;

Loading
Loading