diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 05dc8d8..4d43bcb 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -58,6 +58,7 @@ * [Service](resources/service.md) * [SessionPolicy](resources/session_policy.md) * [Share](resources/share.md) +* [SharedDatabase](resources/shared_database.md) * [SnowflakePartnerOAuthSecurityIntegration](resources/snowflake_partner_oauth_security_integration.md) * [SnowservicesOAuthSecurityIntegration](resources/snowservices_oauth_security_integration.md) * [StageStream](resources/stage_stream.md) diff --git a/docs/resources/grant.md b/docs/resources/grant.md index 81dad6d..765861b 100644 --- a/docs/resources/grant.md +++ b/docs/resources/grant.md @@ -59,6 +59,11 @@ grants: - priv: MONITOR on: cortex search service somedb.someschema.someservice to: search_observability_role + + # IMPORTED PRIVILEGES on a shared database (see SharedDatabase) + - priv: IMPORTED PRIVILEGES + on_database: gong + to: gong_r ``` #### Future Grants @@ -115,6 +120,9 @@ grant = Grant(priv="CREATE TABLE", on_schema="foo", to="somerole") # Table Privileges: grant = Grant(priv=["SELECT", "INSERT", "DELETE"], on_table="sometable", to="somerole") + +# IMPORTED PRIVILEGES on a shared database (see SharedDatabase): +grant = Grant(priv="IMPORTED PRIVILEGES", on_database="gong", to="gong_r") ``` #### Future Grants @@ -196,3 +204,9 @@ grant_on_all = Grant( - **`owner`** (`string` or [Role](role.md), optional): The owner role of the grant. Defaults to `"SYSADMIN"`. + +**Note:** `IMPORTED PRIVILEGES` is only valid on a [SharedDatabase](shared_database.md) +(a database created `FROM SHARE`). It cannot be granted `WITH GRANT OPTION` +and can only be granted to account roles, not database roles. Snowflake's +`SHOW GRANTS` reports it as `USAGE` on shared databases — snowcap's fetch +logic handles this quirk transparently. diff --git a/docs/resources/shared_database.md b/docs/resources/shared_database.md new file mode 100644 index 0000000..c123280 --- /dev/null +++ b/docs/resources/shared_database.md @@ -0,0 +1,97 @@ +--- +description: >- + A database created from an inbound Snowflake share. +--- + +# SharedDatabase + +[Snowflake Documentation](https://docs.snowflake.com/en/sql-reference/sql/create-database) | Snowcap CLI label: `shared database` + +A `SharedDatabase` is the consumer side of a Snowflake share or Marketplace +listing: `CREATE DATABASE FROM SHARE .`. +It's a polymorphic sibling of [Database](database.md) — declared under the +same `databases:` key, distinguished by the presence of `from_share`. Because +Snowflake replicates the provider's schemas, tables, and other objects into +the consumer account, shared databases are read-only: snowcap cannot add +schemas, tags, or params to them the way it can for a regular `Database`. + +## Examples + +### YAML + +```yaml +databases: + - name: gong + from_share: provider_account.share_name + owner: ACCOUNTADMIN +``` + +### Python + +```python +shared_database = SharedDatabase( + name="gong", + from_share="provider_account.share_name", + owner="ACCOUNTADMIN", +) +``` + +## Fields + +* `name` (string, required) - The name of the database. +* `from_share` (string, required) - The `.` the database is created from. Changing this on an existing shared database is not supported by `plan`/`apply` — it errors at plan time. Drop and recreate the database manually instead. +* `owner` (string or [Role](role.md)) - The owner role of the database. Defaults to `"ACCOUNTADMIN"`. + +## Full example: importing a Gong share + +A common pattern is importing a Marketplace or direct share, then handing a +scoped role access to it: + +```yaml +databases: + - name: gong + from_share: provider_account.share_name + +roles: + - name: gong_r + +grants: + - priv: IMPORTED PRIVILEGES + on_database: gong + to: gong_r + +role_grants: + - role: gong_r + to_role: data_engineer +``` + +Which is equivalent to: + +```sql +CREATE DATABASE gong FROM SHARE provider_account.share_name; +CREATE ROLE gong_r; +GRANT IMPORTED PRIVILEGES ON DATABASE gong TO ROLE gong_r; +GRANT ROLE gong_r TO ROLE data_engineer; +``` + +### Gotchas + +- Shared databases are read-only — snowcap does not manage schemas, params, + or tags on them. +- `CREATE DATABASE ... FROM SHARE` requires the account-level `IMPORT SHARE` + privilege, which only `ACCOUNTADMIN` holds by default. Snowcap runs the + creation as `ACCOUNTADMIN` regardless of the configured `owner`. +- `IMPORTED PRIVILEGES` is the only privilege that can be granted on a shared + database. It cannot be granted `WITH GRANT OPTION`, and it can only be + granted to account roles, not database roles. +- Snowflake's `SHOW GRANTS` reports `IMPORTED PRIVILEGES` grants on shared + databases as `USAGE` — snowcap's fetch logic accounts for this quirk, so + `plan`/`apply` still converge correctly. +- Changing `from_share` on an existing shared database is not supported by + `plan`/`apply` — it errors at plan time. Drop and recreate the database + manually if you need to point it at a different share. + +## See also + +- [Database](database.md) — the non-shared sibling resource +- [Grant](grant.md) — for the `IMPORTED PRIVILEGES` grant shown above diff --git a/snowcap/blueprint.py b/snowcap/blueprint.py index 8f82dfb..888e7c7 100644 --- a/snowcap/blueprint.py +++ b/snowcap/blueprint.py @@ -10,6 +10,7 @@ Optional, Sequence, Set, + Type, TypeVar, Union, cast, @@ -42,6 +43,7 @@ ) from .exceptions import ( DuplicateResourceException, + InvalidResourceException, MissingPrivilegeException, MissingResourceException, NonConformingPlanException, @@ -49,7 +51,7 @@ OrphanResourceException, ) from .identifiers import URN, parse_identifier, parse_URN, resource_label_for_type -from .privs import CREATE_PRIV_FOR_RESOURCE_TYPE, system_role_for_priv +from .privs import AccountPriv, CREATE_PRIV_FOR_RESOURCE_TYPE, system_role_for_priv from .resource_name import ResourceName from .resource_tags import ResourceTags from .resources import Database, Grant, RoleGrant, Schema @@ -64,6 +66,7 @@ infer_role_type_from_name, ) from .resources.role import Role +from .resources.shared_database import SharedDatabase from .resources.tag import Tag, TaggableResource from .scope import ( AccountScope, @@ -928,6 +931,23 @@ def _raise_if_plan_would_drop_session_user(session_ctx: SessionContext, plan: Pl raise Exception("Plan would drop the current session user, which is not allowed") +def _require_container(container: Resource, needed_for: Resource) -> ResourceContainer: + """ + SharedDatabase is a DATABASE-typed resource that is deliberately not a ResourceContainer + (shared databases are read-only: no schemas, no db-scoped resources). Every place that + attaches `needed_for` to `container` via .add()/.find() funnels through here, so that case + raises a guided OrphanResourceException instead of a bare AttributeError. + """ + if not isinstance(container, ResourceContainer): + raise OrphanResourceException( + f"Cannot add {needed_for.resource_type.value} '{needed_for.name}' to " + f"{type(container).__name__} '{container.name}': it is read-only and cannot contain " + f"schemas or other resources.\n" + f" Add a regular Database to your config, or set an explicit database:/schema: on the resource." + ) + return container + + def _merge_pointers(resources: Sequence[Resource]) -> list[Resource]: """ It is expected in yaml-defined blueprints that all resources are defined with static strings, instead @@ -939,7 +959,7 @@ def _merge_pointers(resources: Sequence[Resource]) -> list[Resource]: # Push pointers to the end resources = sorted(resources, key=lambda resource: isinstance(resource, ResourcePointer)) - def _merge(resource: ResourceContainer, pointer: ResourcePointer): + def _merge(resource: Resource, pointer: ResourcePointer): if pointer.container is not None: # # The pointer has a container but the resource does not, merge fails # if getattr(resource, "container", None) is None: @@ -949,7 +969,7 @@ def _merge(resource: ResourceContainer, pointer: ResourcePointer): # Migrate items from pointer to resource for item in pointer.items(): pointer.remove(item) - resource.add(item) + _require_container(resource, needed_for=item).add(item) for resource_or_pointer in resources: # Create a unique identifier for the resource @@ -966,8 +986,7 @@ def _merge(resource: ResourceContainer, pointer: ResourcePointer): if isinstance(resource_or_pointer, ResourcePointer): pointer = resource_or_pointer if resource_id in namespace: - primary = cast(ResourceContainer, namespace[resource_id]) - _merge(primary, pointer) + _merge(namespace[resource_id], pointer) else: namespace[resource_id] = pointer else: @@ -1021,6 +1040,21 @@ def _get_role_grants(resource: ResourceContainer) -> list[RoleGrant]: return cast(list[RoleGrant], resource.items(resource_type=ResourceType.ROLE_GRANT)) +def _kind_mismatch_message(urn: URN, declared_cls: Type[Resource], fetched_cls: Type[Resource]) -> str: + hint = "Update your config to match, or drop the resource so it's no longer managed by snowcap." + if {declared_cls, fetched_cls} == {Database, SharedDatabase}: + hint = ( + "declare it with from_share: ." + if fetched_cls is SharedDatabase + else "remove from_share: from its config so it's declared as a regular database" + ) + hint = f"To fix, {hint}." + return ( + f"{urn.fqn.name} is declared as {declared_cls.__name__} but Snowflake reports it as " + f"{fetched_cls.__name__}. {hint}" + ) + + def _resource_scope_is_outside_blueprint_scope(resource_type: ResourceType, blueprint_scope: BlueprintScope) -> bool: resource_scope = RESOURCE_SCOPES[resource_type] if blueprint_scope == BlueprintScope.SCHEMA and ( @@ -1253,11 +1287,20 @@ def _needs_params(urn: URN) -> bool: resource_cls = Resource.resolve_resource_cls(urn.resource_type, data) else: item = manifest[urn] - resource_cls = ( - item.resource_cls - if isinstance(item, ManifestResource) - else Resource.resolve_resource_cls(urn.resource_type, data) - ) + if isinstance(item, ManifestResource): + resource_cls = item.resource_cls + # For polymorphic resource types, the class Snowflake's data actually + # matches can disagree with the class the manifest declared (e.g. a + # database declared as Database that Snowflake reports as an imported + # database). resource_cls.spec(**data) would then raise a raw TypeError + # from an unexpected/missing keyword, so check agreement up front. + fetched_cls = Resource.resolve_resource_cls(urn.resource_type, data) + if fetched_cls is not resource_cls: + raise InvalidResourceException( + _kind_mismatch_message(urn, resource_cls, fetched_cls) + ) + else: + resource_cls = Resource.resolve_resource_cls(urn.resource_type, data) state[urn] = resource_cls.spec(**data).to_dict(session_ctx["account_edition"]) # If data is None, resource doesn't exist in Snowflake # Don't add to state - reconciliation will create it @@ -1362,7 +1405,7 @@ def _build_resource_graph(self, session_ctx: SessionContext) -> None: for resource in db_scoped: if resource.container is None: if len(databases) == 1: - databases[0].add(resource) + _require_container(databases[0], needed_for=resource).add(resource) else: raise OrphanResourceException( f"Resource {resource.resource_type.value} '{resource.name}' has no database.\n" @@ -1373,6 +1416,10 @@ def _build_resource_graph(self, session_ctx: SessionContext) -> None: available_scopes = {} for database in databases: + # SharedDatabase is a DATABASE-typed leaf, not a ResourceContainer: shared + # databases are read-only, so they never have schemas to merge or scope. + if not isinstance(database, ResourceContainer): + continue database_resources = list(database.items()) _merge_pointers(database_resources) for schema in _get_schemas(database): @@ -1383,12 +1430,14 @@ def _build_resource_graph(self, session_ctx: SessionContext) -> None: if len(databases) == 1: # When the blueprint is scoped all dangling resources should be assigned to the configured scope if self._config.scope == BlueprintScope.SCHEMA and self._config.schema is not None: - scoped_schema = _get_schema_by_name(databases[0], self._config.schema) + scoped_schema = _get_schema_by_name( + _require_container(databases[0], needed_for=resource), self._config.schema + ) scoped_schema.add(resource) # TODO: figure out how to handle the case where the schema is already in the blueprint else: logger.warning(f"Resource {resource} has no schema, using {databases[0].name}.PUBLIC") - _get_public_schema(databases[0]).add(resource) + _get_public_schema(_require_container(databases[0], needed_for=resource)).add(resource) else: raise OrphanResourceException( f"Resource {resource.resource_type.value} '{resource.name}' has no schema.\n" @@ -1406,7 +1455,7 @@ def _build_resource_graph(self, session_ctx: SessionContext) -> None: # If the schema pointer has no database, assume it lives in the only database we have if schema_pointer.container is None: if len(databases) == 1: - databases[0].add(schema_pointer) + _require_container(databases[0], needed_for=schema_pointer).add(schema_pointer) else: raise OrphanResourceException( f"Resource {resource.resource_type.value} '{resource.name}' references schema " @@ -1905,7 +1954,12 @@ def execution_strategy_for_change( ) elif isinstance(change, CreateResource): if isinstance(change.resource_cls.scope, AccountScope): - create_priv = CREATE_PRIV_FOR_RESOURCE_TYPE[change.urn.resource_type] + # CREATE DATABASE ... FROM SHARE requires IMPORT SHARE, not CREATE DATABASE + create_priv = ( + AccountPriv.IMPORT_SHARE + if change.resource_cls is SharedDatabase + else CREATE_PRIV_FOR_RESOURCE_TYPE[change.urn.resource_type] + ) # SHARE ownership cannot be changed if change.urn.resource_type == ResourceType.SHARE: @@ -1981,8 +2035,9 @@ def sql_commands_for_change( ) # SPECIAL CASE: when creating a database with a custom owner that we will transfer ownership to, # we also need to transfer ownership of the public schema to that role. This replicates the behavior - # if we were to create the database with a custom owner directly - if change.urn.resource_type == ResourceType.DATABASE: + # if we were to create the database with a custom owner directly. Shared databases have no owned + # PUBLIC schema (Snowflake replicates the provider's schemas read-only), so they're excluded. + if change.urn.resource_type == ResourceType.DATABASE and change.resource_cls is not SharedDatabase: after_change_cmd.append( lifecycle.transfer_resource( public_schema_urn(change.urn), diff --git a/snowcap/data_provider.py b/snowcap/data_provider.py index f556b0e..2b7b5ac 100644 --- a/snowcap/data_provider.py +++ b/snowcap/data_provider.py @@ -1539,7 +1539,10 @@ def fetch_database(session: SnowflakeConnection, fqn: FQN, include_params: bool data = show_result[0] - is_standard_db = data["kind"] in ["STANDARD", "IMPORTED DATABASE"] + if data["kind"] == "IMPORTED DATABASE": + return fetch_shared_database(session, fqn) + + is_standard_db = data["kind"] == "STANDARD" is_snowflake_builtin = data["kind"] == "APPLICATION" and data["name"] in SYSTEM_DATABASES if not (is_standard_db or is_snowflake_builtin): @@ -1908,6 +1911,22 @@ def fetch_grant(session: SnowflakeConnection, fqn: FQN): privilege=priv, role_type=to_type, ) + if data is None and priv == "IMPORTED PRIVILEGES" and on_type == "DATABASE": + # Snowflake reports IMPORTED PRIVILEGES on a shared database as USAGE in SHOW GRANTS. + # Gate on the database actually being shared: without this, a mistakenly-declared + # IMPORTED PRIVILEGES grant on a regular database would false-match its (very common) + # plain USAGE grant and mask the config error. + db_rows = _show_resources(session, "DATABASES", FQN(name=ResourceName(on))) + if db_rows and db_rows[0]["kind"] == "IMPORTED DATABASE": + data = _fetch_grant_to_role( + session, + grant_type=grant_type, + role=to, + granted_on=on_type, + on_name=on, + privilege="USAGE", + role_type=to_type, + ) if data is None: return None privs = [priv] diff --git a/snowcap/privs.py b/snowcap/privs.py index 4d6ca8c..82ca383 100644 --- a/snowcap/privs.py +++ b/snowcap/privs.py @@ -99,7 +99,7 @@ class DatabasePriv(Priv): APPLYBUDGET = "APPLYBUDGET" CREATE_DATABASE_ROLE = "CREATE DATABASE ROLE" CREATE_SCHEMA = "CREATE SCHEMA" - # IMPORTED_PRIVILEGES = "IMPORTED PRIVILEGES" # only granted on shared database + IMPORTED_PRIVILEGES = "IMPORTED PRIVILEGES" # only granted on shared databases MODIFY = "MODIFY" MONITOR = "MONITOR" OWNERSHIP = "OWNERSHIP" @@ -560,7 +560,9 @@ def all_privs_for_resource_type(resource_type): privs = PRIVS_FOR_RESOURCE_TYPE[resource_type] or [] for priv in privs: priv = str(priv) - if priv != "ALL" and priv != "OWNERSHIP": + # IMPORTED PRIVILEGES is only grantable on shared databases; excluding it here keeps + # priv="ALL" grants on regular databases from expanding into an invalid grant. + if priv not in ("ALL", "OWNERSHIP", "IMPORTED PRIVILEGES"): all_privs.append(priv) return all_privs diff --git a/snowcap/resources/__init__.py b/snowcap/resources/__init__.py index 2406891..778341c 100644 --- a/snowcap/resources/__init__.py +++ b/snowcap/resources/__init__.py @@ -58,6 +58,7 @@ from .sequence import Sequence from .service import Service from .share import Share +from .shared_database import SharedDatabase from .stage import ExternalStage, InternalStage from .storage_integration import ( AzureStorageIntegration, @@ -139,6 +140,7 @@ "Sequence", "Service", "Share", + "SharedDatabase", "SnowflakeIcebergTable", "SnowflakePartnerOAuthSecurityIntegration", "SnowservicesOAuthSecurityIntegration", diff --git a/snowcap/resources/shared_database.py b/snowcap/resources/shared_database.py index 85dbbcc..d498985 100644 --- a/snowcap/resources/shared_database.py +++ b/snowcap/resources/shared_database.py @@ -1,40 +1,88 @@ -# from dataclasses import dataclass - -# from .resource import Resource, ResourceSpec -# from ..enums import ResourceType -# from ..scope import AccountScope -# from ..props import Props, IdentifierProp - - -# @dataclass(unsafe_hash=True) -# class _SharedDatabase(ResourceSpec): -# name: str -# from_share: str -# owner: str = "ACCOUNTADMIN" - - -# class SharedDatabase(Resource): -# """ -# CREATE DATABASE FROM SHARE . -# """ - -# resource_type = ResourceType.DATABASE -# props = Props( -# from_share=IdentifierProp("from share", eq=False), -# ) -# scope = AccountScope() -# spec = _SharedDatabase - -# def __init__( -# self, -# name: str, -# from_share: str, -# owner: str = "ACCOUNTADMIN", -# **kwargs, -# ): -# super().__init__(**kwargs) -# self._data = _SharedDatabase( -# name=name, -# from_share=from_share, -# owner=owner, -# ) +from dataclasses import dataclass, field + +from ..enums import ResourceType +from ..props import IdentifierProp, Props +from ..resource_name import ResourceName +from ..scope import AccountScope +from .database import Database +from .resource import NamedResource, Resource, ResourceSpec +from .role import Role + +# ResourceType.DATABASE is polymorphic: it resolves to either Database (database.py) or +# SharedDatabase (this file), depending on whether `from_share` is present in the data. The +# resolver is registered here, alongside SharedDatabase, rather than in database.py, so anyone +# grepping for the DATABASE subtypes finds this pointer. + + +@dataclass(unsafe_hash=True) +class _SharedDatabase(ResourceSpec): + name: ResourceName + from_share: ResourceName = field(metadata={"triggers_replacement": True}) + owner: Role = "ACCOUNTADMIN" + + +class SharedDatabase(NamedResource, Resource): + """ + Description: + A database created from a Snowflake share. Shared databases are read-only: Snowflake + replicates the provider's schemas, tables, and other objects into the consumer account, + so snowcap cannot add schemas, tags, or params to them the way it can for a regular + Database. + + Snowflake Docs: + https://docs.snowflake.com/en/sql-reference/sql/create-database#create-database-from-share + + Fields: + name (string, required): The name of the database. + from_share (string, required): The `.` the database is created from. + owner (string or Role): The owner role of the database. Defaults to "ACCOUNTADMIN". + + Python: + + ```python + shared_database = SharedDatabase( + name="gong", + from_share="provider_account.share_name", + owner="ACCOUNTADMIN", + ) + ``` + + Yaml: + + ```yaml + databases: + - name: gong + from_share: provider_account.share_name + owner: ACCOUNTADMIN + ``` + """ + + resource_type = ResourceType.DATABASE + props = Props( + from_share=IdentifierProp("from share", eq=False), + ) + scope = AccountScope() + spec = _SharedDatabase + + def __init__( + self, + name: str, + from_share: str, + owner: str = "ACCOUNTADMIN", + **kwargs, + ): + super().__init__(name, **kwargs) + self._data: _SharedDatabase = _SharedDatabase( + name=self._name, + from_share=from_share, + owner=owner, + ) + + +# Discriminates on field presence, like stream.py's resolver -- not an enum map like +# stage.py's StageTypeMap, since there's no explicit "type" field to key off of. +def _resolver(data: dict): + return SharedDatabase if data.get("from_share") else Database + + +Resource.__resolvers__[ResourceType.DATABASE] = _resolver diff --git a/tests/fixtures/json/shared_database.json b/tests/fixtures/json/shared_database.json new file mode 100644 index 0000000..81d319c --- /dev/null +++ b/tests/fixtures/json/shared_database.json @@ -0,0 +1,5 @@ +{ + "name": "TEST_SHARED_DATABASE", + "from_share": "PROVIDER_ACCOUNT.SHARE_NAME", + "owner": "ACCOUNTADMIN" +} \ No newline at end of file diff --git a/tests/fixtures/sql/shared_database.sql b/tests/fixtures/sql/shared_database.sql new file mode 100644 index 0000000..10e3e85 --- /dev/null +++ b/tests/fixtures/sql/shared_database.sql @@ -0,0 +1 @@ +CREATE DATABASE mytestdb FROM SHARE myprovider.myshare; diff --git a/tests/test_blueprint.py b/tests/test_blueprint.py index 75c3045..081d645 100644 --- a/tests/test_blueprint.py +++ b/tests/test_blueprint.py @@ -42,7 +42,7 @@ def flatten_sql_commands(sql_commands_result) -> list[str]: return result -from snowcap import var +from snowcap import data_provider, var from snowcap.blueprint import ( Blueprint, CreateResource, @@ -55,8 +55,10 @@ def flatten_sql_commands(sql_commands_result) -> list[str]: from snowcap.enums import AccountEdition, BlueprintScope, ResourceType from snowcap.exceptions import ( DuplicateResourceException, + InvalidResourceException, MissingVarException, NonConformingPlanException, + OrphanResourceException, WrongEditionException, ) from snowcap.identifiers import FQN, URN, parse_URN @@ -1099,3 +1101,185 @@ def test_resource_type_needs_params(session_ctx): ) manifest = blueprint.generate_manifest(session_ctx) assert resource_type_needs_params(ResourceType.ROLE, manifest) is True + + +def test_blueprint_shared_database_create_default_owner(session_ctx, remote_state): + shared_db = res.SharedDatabase(name="GONG", from_share="provider_account.share_name") + blueprint = Blueprint(name="blueprint", resources=[shared_db]) + manifest = blueprint.generate_manifest(session_ctx) + plan = diff(remote_state, manifest) + assert len(plan) == 1 + assert plan[0].resource_cls == res.SharedDatabase + + commands = flatten_sql_commands(compile_plan_to_sql(session_ctx, plan)) + assert "USE ROLE ACCOUNTADMIN" in commands + assert any(c.startswith("CREATE DATABASE") and "FROM SHARE" in c for c in commands) + assert not any(c.startswith("GRANT OWNERSHIP") for c in commands) + + +def test_blueprint_shared_database_create_custom_owner_skips_public_schema_transfer(session_ctx, remote_state): + shared_db = res.SharedDatabase(name="GONG", from_share="provider_account.share_name", owner="SYSADMIN") + blueprint = Blueprint(name="blueprint", resources=[shared_db]) + manifest = blueprint.generate_manifest(session_ctx) + plan = diff(remote_state, manifest) + + commands = flatten_sql_commands(compile_plan_to_sql(session_ctx, plan)) + assert "GRANT OWNERSHIP ON DATABASE GONG TO ROLE SYSADMIN COPY CURRENT GRANTS" in commands + assert not any("PUBLIC" in c for c in commands) + + +def test_blueprint_database_create_custom_owner_transfers_public_schema(session_ctx, remote_state): + # Regression: a regular Database with a non-default owner still transfers the PUBLIC schema. + db = res.Database(name="DB", owner="USERADMIN") + blueprint = Blueprint(name="blueprint", resources=[db]) + manifest = blueprint.generate_manifest(session_ctx) + plan = diff(remote_state, manifest) + + commands = flatten_sql_commands(compile_plan_to_sql(session_ctx, plan)) + assert "GRANT OWNERSHIP ON DATABASE DB TO ROLE USERADMIN COPY CURRENT GRANTS" in commands + assert "GRANT OWNERSHIP ON SCHEMA DB.PUBLIC TO ROLE USERADMIN COPY CURRENT GRANTS" in commands + + +def test_blueprint_shared_database_from_share_change_raises_not_implemented(session_ctx, remote_state): + # Documents today's limitation: from_share triggers_replacement, and replace_resource + # is unimplemented, so a from_share drift is a plan-time error rather than a silent + # (and nonsensical) ALTER at apply time. + remote_state[parse_URN("urn::ABCD123:database/GONG")] = { + "name": "GONG", + "from_share": "provider_account.old_share", + "owner": "ACCOUNTADMIN", + } + shared_db = res.SharedDatabase(name="GONG", from_share="provider_account.new_share") + blueprint = Blueprint(name="blueprint", resources=[shared_db]) + manifest = blueprint.generate_manifest(session_ctx) + + with pytest.raises(NotImplementedError, match="replace_resource"): + diff(remote_state, manifest) + + +def test_blueprint_shared_database_idempotent_round_trip(session_ctx, remote_state): + # Remote state mirrors what fetch_shared_database returns: Snowflake normalizes + # unquoted identifiers to uppercase, matching the manifest's own normalization. + remote_state[parse_URN("urn::ABCD123:database/GONG")] = { + "name": "GONG", + "from_share": "PROVIDER_ACCOUNT.SHARE_NAME", + "owner": "ACCOUNTADMIN", + } + shared_db = res.SharedDatabase(name="GONG", from_share="provider_account.share_name") + blueprint = Blueprint(name="blueprint", resources=[shared_db]) + manifest = blueprint.generate_manifest(session_ctx) + plan = diff(remote_state, manifest) + + assert plan == [] + + +def _fetch_remote_state_with_mocked_fetch(blueprint, manifest, session_ctx, monkeypatch, fetched_data): + """Drive Blueprint.fetch_remote_state with data_provider entirely mocked out. + + DATABASE-typed URNs get `fetched_data`; ROLE references (e.g. the database's owner) are + reported as existing so the post-fetch reference check passes; anything else (e.g. the + implicit ACCOUNT resource) is reported as not-found and simply skipped. + """ + monkeypatch.setattr(data_provider, "fetch_session", lambda session: session_ctx) + monkeypatch.setattr(data_provider, "use_secondary_roles", lambda session, all=False: None) + + def _fetch_resource(session, urn, include_params=True, existence_only=False): + if urn.resource_type == ResourceType.DATABASE: + return fetched_data + if urn.resource_type == ResourceType.ROLE: + return {"name": str(urn.fqn.name)} + return None + + monkeypatch.setattr(data_provider, "fetch_resource", _fetch_resource) + return blueprint.fetch_remote_state(session=None, manifest=manifest) + + +def test_fetch_remote_state_declared_database_but_remote_is_shared_raises_clear_error(session_ctx, monkeypatch): + # Regression for FINDING 1: a pre-existing config declares a plain Database, but Snowflake + # reports the same URN as an imported (shared) database. This must raise a guided domain + # error, not a raw TypeError from Database(**data) choking on the unexpected 'from_share' key. + db = res.Database(name="GONG", owner="SYSADMIN") + blueprint = Blueprint(name="blueprint", resources=[db]) + manifest = blueprint.generate_manifest(session_ctx) + + fetched_data = {"name": "GONG", "from_share": "provider_account.share_name", "owner": "ACCOUNTADMIN"} + with pytest.raises( + InvalidResourceException, match="declared as Database but Snowflake reports it as SharedDatabase" + ): + _fetch_remote_state_with_mocked_fetch(blueprint, manifest, session_ctx, monkeypatch, fetched_data) + + +def test_fetch_remote_state_declared_shared_database_but_remote_is_standard_raises_clear_error( + session_ctx, monkeypatch +): + # Reverse of the above: declared as SharedDatabase, but Snowflake reports a STANDARD database. + shared_db = res.SharedDatabase(name="GONG", from_share="provider_account.share_name") + blueprint = Blueprint(name="blueprint", resources=[shared_db]) + manifest = blueprint.generate_manifest(session_ctx) + + fetched_data = {"name": "GONG", "owner": "SYSADMIN"} + with pytest.raises( + InvalidResourceException, match="declared as SharedDatabase but Snowflake reports it as Database" + ): + _fetch_remote_state_with_mocked_fetch(blueprint, manifest, session_ctx, monkeypatch, fetched_data) + + +def test_fetch_remote_state_matching_declared_and_fetched_kind_does_not_raise(session_ctx, monkeypatch): + # Regression: when the declared and fetched classes agree, fetch_remote_state proceeds normally. + db = res.Database(name="GONG", owner="SYSADMIN") + blueprint = Blueprint(name="blueprint", resources=[db]) + manifest = blueprint.generate_manifest(session_ctx) + + fetched_data = {"name": "GONG", "owner": "SYSADMIN"} + state = _fetch_remote_state_with_mocked_fetch(blueprint, manifest, session_ctx, monkeypatch, fetched_data) + assert len(state) == 1 + + +def test_shared_database_sole_db_scoped_resource_raises_clear_error(session_ctx): + # Regression for FINDING 2 (site: databases[0].add(resource) for parentless db-scoped + # resources): a SharedDatabase is not a ResourceContainer, so a parentless Schema (no + # database: set) must raise a guided OrphanResourceException instead of AttributeError. + shared_db = res.SharedDatabase(name="GONG", from_share="provider_account.share_name") + schema = res.Schema(name="MY_SCHEMA") + blueprint = Blueprint(name="blueprint", resources=[shared_db, schema]) + + with pytest.raises(OrphanResourceException, match="Cannot add SCHEMA 'MY_SCHEMA' to SharedDatabase 'GONG'"): + blueprint.generate_manifest(session_ctx) + + +def test_shared_database_sole_public_schema_target_raises_clear_error(session_ctx): + # Regression for FINDING 2 (site: _get_public_schema(databases[0]) for parentless + # schema-scoped resources): a Table with no schema:/database: set falls back to + # ".PUBLIC", which doesn't exist on a read-only SharedDatabase. + shared_db = res.SharedDatabase(name="GONG", from_share="provider_account.share_name") + table = res.Table(name="MY_TABLE", columns=[{"name": "ID", "data_type": "INT"}]) + blueprint = Blueprint(name="blueprint", resources=[shared_db, table]) + + with pytest.raises(OrphanResourceException, match="Cannot add TABLE 'MY_TABLE' to SharedDatabase 'GONG'"): + blueprint.generate_manifest(session_ctx) + + +def test_shared_database_schema_pointer_without_database_raises_clear_error(session_ctx): + # Regression for FINDING 2 (site: databases[0].add(schema_pointer) for a schema-scoped + # resource that names its schema but not its database): the implied schema pointer would + # be attached to the sole database, which cannot hold it if that database is shared. + shared_db = res.SharedDatabase(name="GONG", from_share="provider_account.share_name") + table = res.Table(name="MY_TABLE", schema="SOME_SCHEMA", columns=[{"name": "ID", "data_type": "INT"}]) + blueprint = Blueprint(name="blueprint", resources=[shared_db, table]) + + with pytest.raises(OrphanResourceException, match="Cannot add SCHEMA 'SOME_SCHEMA' to SharedDatabase 'GONG'"): + blueprint.generate_manifest(session_ctx) + + +def test_schema_under_shared_database_raises_clear_error(session_ctx): + # Regression for FINDING 3: Schema(database='gong') registers a ResourcePointer that + # _merge_pointers later merges into the resolved SharedDatabase resource. Since + # SharedDatabase is not a ResourceContainer, this is a different code path from FINDING 2 + # (explicitly-parented vs. parentless) and must independently raise a guided error instead + # of a bare AttributeError deep in merge internals. + shared_db = res.SharedDatabase(name="gong", from_share="provider_account.share_name") + schema = res.Schema(name="foo", database="gong") + blueprint = Blueprint(name="blueprint", resources=[shared_db, schema]) + + with pytest.raises(OrphanResourceException, match="Cannot add SCHEMA '.*' to SharedDatabase 'GONG'"): + blueprint.generate_manifest(session_ctx) diff --git a/tests/test_data_provider.py b/tests/test_data_provider.py index 2368656..9ecff46 100644 --- a/tests/test_data_provider.py +++ b/tests/test_data_provider.py @@ -33,6 +33,8 @@ remove_none_values, # Dispatcher functions fetch_resource, + fetch_database, + fetch_grant, fetch_warehouse, list_resource, list_account_scoped_resource, @@ -642,6 +644,153 @@ def test_raises_other_programming_errors(self, mock_fetch_role): fetch_resource(mock_session, urn) +def _database_show_row(**overrides): + row = { + "name": "MY_DB", + "kind": "STANDARD", + "owner": "SYSADMIN", + "owner_role_type": "ROLE", + "retention_time": "1", + "comment": "", + "options": "", + } + row.update(overrides) + return row + + +class TestFetchDatabase: + """Tests for fetch_database's imported-database delegation.""" + + @patch("snowcap.data_provider.fetch_shared_database") + @patch("snowcap.data_provider._show_resource_parameters") + @patch("snowcap.data_provider._show_resources") + def test_imported_database_delegates_to_fetch_shared_database( + self, mock_show_resources, mock_show_params, mock_fetch_shared_database + ): + mock_show_resources.return_value = [_database_show_row(kind="IMPORTED DATABASE", name="GONG")] + mock_fetch_shared_database.return_value = { + "name": "GONG", + "from_share": "provider_account.share_name", + "owner": "ACCOUNTADMIN", + } + mock_session = MagicMock() + fqn = FQN(name=ResourceName("GONG")) + + result = fetch_database(mock_session, fqn) + + mock_fetch_shared_database.assert_called_once_with(mock_session, fqn) + mock_show_params.assert_not_called() + assert result == mock_fetch_shared_database.return_value + assert "data_retention_time_in_days" not in result + + @patch("snowcap.data_provider._show_resource_parameters") + @patch("snowcap.data_provider._show_resources") + def test_standard_database_returns_full_shape(self, mock_show_resources, mock_show_params): + mock_show_resources.return_value = [_database_show_row()] + mock_show_params.return_value = {"default_ddl_collation": ""} + mock_session = MagicMock() + + result = fetch_database(mock_session, FQN(name=ResourceName("MY_DB"))) + + assert result["name"] == "MY_DB" + assert result["data_retention_time_in_days"] == 1 + assert result["owner"] == "SYSADMIN" + assert result["transient"] is False + + +def _imported_privileges_grant_fqn(): + return FQN( + name=ResourceName("GRANT"), + params={"priv": "IMPORTED PRIVILEGES", "on": "database/gong", "to": "role/gong_r"}, + ) + + +def _grant_to_role_row(**overrides): + row = { + "created_on": "2024-01-01", + "privilege": "USAGE", + "granted_on": "DATABASE", + "name": "GONG", + "granted_to": "ROLE", + "grantee_name": "GONG_R", + "grant_option": "false", + "granted_by": "ACCOUNTADMIN", + } + row.update(overrides) + return row + + +class TestFetchGrantImportedPrivilegesQuirk: + """Tests for fetch_grant's IMPORTED PRIVILEGES/USAGE reporting quirk on shared databases.""" + + @patch("snowcap.data_provider._show_resources") + @patch("snowcap.data_provider._show_grants_to_role") + def test_matches_when_reported_as_usage_on_imported_database(self, mock_show_grants, mock_show_resources): + mock_show_grants.return_value = [_grant_to_role_row(privilege="USAGE")] + mock_show_resources.return_value = [_database_show_row(kind="IMPORTED DATABASE", name="GONG")] + mock_session = MagicMock() + + result = fetch_grant(mock_session, _imported_privileges_grant_fqn()) + + assert result is not None + assert result["priv"] == "IMPORTED PRIVILEGES" + + @patch("snowcap.data_provider._show_resources") + @patch("snowcap.data_provider._show_grants_to_role") + def test_matches_when_reported_verbatim(self, mock_show_grants, mock_show_resources): + mock_show_grants.return_value = [_grant_to_role_row(privilege="IMPORTED PRIVILEGES")] + mock_session = MagicMock() + + result = fetch_grant(mock_session, _imported_privileges_grant_fqn()) + + assert result is not None + assert result["priv"] == "IMPORTED PRIVILEGES" + # Exact match succeeded, so the USAGE fallback (and its kind check) never ran. + mock_show_resources.assert_not_called() + + @patch("snowcap.data_provider._show_resources") + @patch("snowcap.data_provider._show_grants_to_role") + def test_returns_none_when_grant_genuinely_absent(self, mock_show_grants, mock_show_resources): + mock_show_grants.return_value = [] + mock_show_resources.return_value = [_database_show_row(kind="IMPORTED DATABASE", name="GONG")] + mock_session = MagicMock() + + result = fetch_grant(mock_session, _imported_privileges_grant_fqn()) + + assert result is None + + @patch("snowcap.data_provider._show_resources") + @patch("snowcap.data_provider._show_grants_to_role") + def test_plain_usage_grant_on_regular_database_unaffected(self, mock_show_grants, mock_show_resources): + mock_show_grants.return_value = [_grant_to_role_row(privilege="USAGE")] + mock_session = MagicMock() + fqn = FQN( + name=ResourceName("GRANT"), + params={"priv": "USAGE", "on": "database/gong", "to": "role/gong_r"}, + ) + + result = fetch_grant(mock_session, fqn) + + assert result is not None + assert result["priv"] == "USAGE" + # Exact match succeeded for the requested privilege, so the imported-database + # kind check (which only applies to the IMPORTED PRIVILEGES fallback) never ran. + mock_show_resources.assert_not_called() + + @patch("snowcap.data_provider._show_resources") + @patch("snowcap.data_provider._show_grants_to_role") + def test_does_not_false_match_usage_on_regular_database(self, mock_show_grants, mock_show_resources): + # A REGULAR database with a plain USAGE grant must not be mistaken for an + # IMPORTED PRIVILEGES grant -- that would mask a genuine config error. + mock_show_grants.return_value = [_grant_to_role_row(privilege="USAGE")] + mock_show_resources.return_value = [_database_show_row(kind="STANDARD", name="GONG")] + mock_session = MagicMock() + + result = fetch_grant(mock_session, _imported_privileges_grant_fqn()) + + assert result is None + + def _warehouse_show_row(**overrides): row = { "name": "WH", diff --git a/tests/test_privs.py b/tests/test_privs.py index 6450721..276edca 100644 --- a/tests/test_privs.py +++ b/tests/test_privs.py @@ -202,6 +202,11 @@ def test_monitor_privilege(self): """DatabasePriv has MONITOR privilege.""" assert DatabasePriv.MONITOR.value == "MONITOR" + def test_imported_privileges_privilege(self): + """DatabasePriv has IMPORTED PRIVILEGES privilege, granted on shared databases.""" + assert DatabasePriv.IMPORTED_PRIVILEGES.value == "IMPORTED PRIVILEGES" + assert DatabasePriv("IMPORTED PRIVILEGES") == DatabasePriv.IMPORTED_PRIVILEGES + ############################################################################# # SchemaPriv Tests @@ -395,6 +400,12 @@ def test_database_privs(self): assert "ALL" not in privs assert "OWNERSHIP" not in privs + def test_database_privs_exclude_imported_privileges(self): + """all_privs_for_resource_type excludes IMPORTED PRIVILEGES so priv='ALL' on a regular + database never expands to a privilege that's only grantable on shared databases.""" + privs = all_privs_for_resource_type(ResourceType.DATABASE) + assert "IMPORTED PRIVILEGES" not in privs + def test_table_privs(self): """all_privs_for_resource_type returns non-ALL/OWNERSHIP privileges for table.""" privs = all_privs_for_resource_type(ResourceType.TABLE) @@ -518,6 +529,12 @@ def test_from_grant_with_no_priv_type(self): assert gp.privilege == "SOME_PRIV" assert gp.on == "GRANT" + def test_from_grant_with_imported_privileges_on_database(self): + """GrantedPrivilege.from_grant works for IMPORTED PRIVILEGES granted on a shared database.""" + gp = GrantedPrivilege.from_grant(privilege="IMPORTED PRIVILEGES", granted_on="DATABASE", name="SOME_SHARED_DB") + assert gp.privilege == DatabasePriv.IMPORTED_PRIVILEGES + assert gp.on == "SOME_SHARED_DB" + def test_repr(self): """GrantedPrivilege has a useful repr.""" gp = GrantedPrivilege(privilege=TablePriv.SELECT, on="MY_TABLE") diff --git a/tests/test_resource_types.py b/tests/test_resource_types.py index 1d120a1..4321ea9 100644 --- a/tests/test_resource_types.py +++ b/tests/test_resource_types.py @@ -17,6 +17,7 @@ from snowcap.enums import ResourceType, WarehouseSize from snowcap.identifiers import FQN, URN from snowcap.resource_name import ResourceName +from snowcap.resources.resource import Resource, ResourceContainer class TestDatabase: @@ -80,6 +81,65 @@ def test_database_to_dict(self): assert data["comment"] == "Test" +class TestSharedDatabase: + """Tests for SharedDatabase resource, the polymorphic sibling of Database for + databases created FROM SHARE.""" + + def test_shared_database_minimal(self): + """Test SharedDatabase with minimal required properties.""" + db = res.SharedDatabase(name="gong", from_share="provider_account.share_name") + assert db.name == "gong" + assert db.resource_type == ResourceType.DATABASE + + def test_shared_database_default_owner_is_accountadmin(self): + """SharedDatabase defaults to ACCOUNTADMIN ownership, unlike Database's SYSADMIN default.""" + db = res.SharedDatabase(name="gong", from_share="provider_account.share_name") + assert str(db._data.owner.name) == "ACCOUNTADMIN" + + def test_shared_database_create_sql(self): + """SharedDatabase.create_sql() renders CREATE DATABASE ... FROM SHARE ...""" + db = res.SharedDatabase(name="gong", from_share="provider_account.share_name") + sql = db.create_sql() + assert sql == "CREATE DATABASE GONG FROM SHARE PROVIDER_ACCOUNT.SHARE_NAME" + + def test_shared_database_is_not_a_resource_container(self): + """SharedDatabase is read-only (no schemas, no tags, no params) so it must not be a + ResourceContainer.""" + assert not issubclass(res.SharedDatabase, ResourceContainer) + + def test_shared_database_resolver_picks_shared_database(self): + """Resource.resolve_resource_cls resolves to SharedDatabase when from_share is present.""" + resource_cls = Resource.resolve_resource_cls(ResourceType.DATABASE, {"name": "gong", "from_share": "a.b"}) + assert resource_cls is res.SharedDatabase + + def test_shared_database_resolver_picks_database(self): + """Resource.resolve_resource_cls resolves to Database when from_share is absent.""" + resource_cls = Resource.resolve_resource_cls(ResourceType.DATABASE, {"name": "analytics"}) + assert resource_cls is res.Database + + @pytest.mark.xfail( + reason=( + "ResourceName wraps from_share as a single opaque string, so a compound identifier " + "with one quoted component (e.g. SOME_ORG.\"My Share\") gets uppercased wholesale by " + "ResourceName.__str__ instead of preserving the quoted part. ResourceName has no " + "concept of compound (dotted) identifiers with independently-quoted components -- " + "this is a pre-existing limitation shared by every IdentifierProp field, not " + "something introduced by SharedDatabase, so fixing it is out of scope here." + ), + strict=True, + ) + def test_shared_database_from_share_quoted_component_round_trips(self): + """A from_share value with a quoted component should preserve that quoting through + create_sql, and from_sql should parse it back without mangling it.""" + db = res.SharedDatabase(name="gong", from_share='SOME_ORG."My Share"') + assert db.create_sql() == 'CREATE DATABASE GONG FROM SHARE SOME_ORG."My Share"' + + round_tripped = res.SharedDatabase.from_sql( + 'CREATE DATABASE gong FROM SHARE SOME_ORG."My Share"' + ) + assert round_tripped._data.from_share == 'SOME_ORG."My Share"' + + class TestSchema: """Tests for Schema resource.""" diff --git a/tests/test_yaml_config.py b/tests/test_yaml_config.py index 52cee98..3d06af3 100644 --- a/tests/test_yaml_config.py +++ b/tests/test_yaml_config.py @@ -635,3 +635,51 @@ def test_all_fixtures_can_be_loaded_by_parser(self): config = self._load_fixture(os.path.basename(fixture_path)) blueprint_config = collect_blueprint_config(config) assert len(blueprint_config.resources) > 0, f"{fixture_path} produced no resources" + + +class TestSharedDatabaseYaml: + """Tests for the YAML surface of shared databases and their IMPORTED PRIVILEGES grants: + a `databases:` entry with `from_share` resolves to SharedDatabase, and a `grants:` entry + can request IMPORTED PRIVILEGES on it.""" + + def test_database_with_from_share_produces_shared_database(self): + """Test: a databases: entry with from_share: resolves to SharedDatabase, not Database.""" + from snowcap import resources as res + + config = { + "databases": [ + {"name": "gong", "from_share": "provider_account.share_name", "owner": "ACCOUNTADMIN"}, + ], + } + blueprint_config = collect_blueprint_config(config) + + assert len(blueprint_config.resources) == 1 + assert isinstance(blueprint_config.resources[0], res.SharedDatabase) + + def test_database_without_from_share_still_produces_database(self): + """Test: a databases: entry without from_share: still resolves to the regular Database + (regression check now that ResourceType.DATABASE is polymorphic).""" + from snowcap import resources as res + + config = {"databases": [{"name": "analytics"}]} + blueprint_config = collect_blueprint_config(config) + + assert len(blueprint_config.resources) == 1 + assert isinstance(blueprint_config.resources[0], res.Database) + + def test_imported_privileges_grant_create_sql(self): + """Test: a grants: entry for IMPORTED PRIVILEGES on a database parses into a Grant with + the expected create SQL.""" + from snowcap import resources as res + + config = { + "grants": [ + {"priv": "IMPORTED PRIVILEGES", "on_database": "gong", "to": "gong_r"}, + ], + } + blueprint_config = collect_blueprint_config(config) + + assert len(blueprint_config.resources) == 1 + grant = blueprint_config.resources[0] + assert isinstance(grant, res.Grant) + assert grant.create_sql() == "GRANT IMPORTED PRIVILEGES ON DATABASE GONG TO ROLE GONG_R"