diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 90102e0a3..153783450 100755 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -bbafbee4a953dd371abfc51ab8806d33eba2b734 \ No newline at end of file +11ae6f9d98f0d0838a5e53c27032f178fecc4ee0 \ No newline at end of file diff --git a/.github/workflows/next-changelog.yml b/.github/workflows/next-changelog.yml index b321edd6e..847aadb8e 100755 --- a/.github/workflows/next-changelog.yml +++ b/.github/workflows/next-changelog.yml @@ -4,16 +4,19 @@ name: Check for NEXT_CHANGELOG.md Changes on: # Use pull_request_target to have access to GitHub API pull_request_target: + types: [opened, synchronize, reopened, edited] jobs: check-next-changelog: + # Allow Dependabot PRs to pass without a changelog entry + if: github.actor != 'dependabot[bot]' runs-on: group: databricks-deco-testing-runner-group labels: ubuntu-latest-deco steps: - name: Checkout code - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Fetch list of changed files id: changed-files diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 1b9a3ae70..83fdf71c0 100755 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -30,3 +30,34 @@ * Add `default_branch` field for `databricks.sdk.service.postgres.ProjectSpec`. * Add `default_branch` field for `databricks.sdk.service.postgres.ProjectStatus`. * Add `ingress` and `ingress_dry_run` fields for `databricks.sdk.service.settings.AccountNetworkPolicy`. +* Add `delete_app_thumbnail()` and `update_app_thumbnail()` methods for [w.apps](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/apps/apps.html) workspace-level service. +* Add `create_message_comment()`, `list_conversation_comments()` and `list_message_comments()` methods for [w.genie](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/dashboards/genie.html) workspace-level service. +* Add `apply_environment()` method for [w.pipelines](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/pipelines/pipelines.html) workspace-level service. +* Add `name` and `permission` fields for `databricks.sdk.service.apps.AppResourceApp`. +* Add `managed_encryption_settings` field for `databricks.sdk.service.catalog.CatalogInfo`. +* Add `managed_encryption_settings` field for `databricks.sdk.service.catalog.CreateCatalog`. +* Add `managed_encryption_settings` field for `databricks.sdk.service.catalog.UpdateCatalog`. +* Add `comment` field for `databricks.sdk.service.dashboards.GenieFeedback`. +* Add `thoughts` field for `databricks.sdk.service.dashboards.GenieQueryAttachment`. +* Add `comment` field for `databricks.sdk.service.dashboards.GenieSendMessageFeedbackRequest`. +* Add `request_source` field for `databricks.sdk.service.ml.DataSource`. +* Add `is_online` field for `databricks.sdk.service.ml.MaterializedFeature`. +* Add `connector_options` field for `databricks.sdk.service.pipelines.SchemaSpec`. +* Add `connector_options` field for `databricks.sdk.service.pipelines.TableSpec`. +* Add `scopes` field for `databricks.sdk.service.settings.CreateOboTokenRequest`. +* Add `gcp_endpoint` field for `databricks.sdk.service.settings.CreatePrivateEndpointRule`. +* Add `scopes` field for `databricks.sdk.service.settings.CreateTokenRequest`. +* Add `gcp_endpoint` field for `databricks.sdk.service.settings.NccPrivateEndpointRule`. +* Add `gcp_endpoint` field for `databricks.sdk.service.settings.UpdatePrivateEndpointRule`. +* Add `sql_state` field for `databricks.sdk.service.sql.StatementStatus`. +* Add `usage_policy_id` field for `databricks.sdk.service.vectorsearch.CreateEndpoint`. +* Add `index_subtype` field for `databricks.sdk.service.vectorsearch.CreateVectorIndexRequest`. +* Add `budget_policy_id` field for `databricks.sdk.service.vectorsearch.EndpointInfo`. +* Add `index_subtype` field for `databricks.sdk.service.vectorsearch.MiniVectorIndex`. +* Add `budget_policy_id` field for `databricks.sdk.service.vectorsearch.PatchEndpointBudgetPolicyResponse`. +* Add `index_subtype` field for `databricks.sdk.service.vectorsearch.VectorIndex`. +* Add `table_delta_uniform_iceberg_external_deltasharing` enum value for `databricks.sdk.service.catalog.SecurableKind`. +* Add `google_drive` enum value for `databricks.sdk.service.pipelines.IngestionSourceType`. +* Add `storage_optimized` enum value for `databricks.sdk.service.vectorsearch.EndpointType`. +* [Breaking] Remove `project` field for `databricks.sdk.service.postgres.SyncedTableSyncedTableSpec`. +* [Breaking] Remove `apps` and `lakebase` fields for `databricks.sdk.service.settings.CustomerFacingIngressNetworkPolicyRequestDestination`. \ No newline at end of file diff --git a/databricks/sdk/service/apps.py b/databricks/sdk/service/apps.py index 63b32d072..caaed43c6 100755 --- a/databricks/sdk/service/apps.py +++ b/databricks/sdk/service/apps.py @@ -1139,20 +1139,37 @@ def from_dict(cls, d: Dict[str, Any]) -> AppResource: @dataclass class AppResourceApp: + name: Optional[str] = None + + permission: Optional[AppResourceAppAppPermission] = None + def as_dict(self) -> dict: """Serializes the AppResourceApp into a dictionary suitable for use as a JSON request body.""" body = {} + if self.name is not None: + body["name"] = self.name + if self.permission is not None: + body["permission"] = self.permission.value return body def as_shallow_dict(self) -> dict: """Serializes the AppResourceApp into a shallow dictionary of its immediate attributes.""" body = {} + if self.name is not None: + body["name"] = self.name + if self.permission is not None: + body["permission"] = self.permission return body @classmethod def from_dict(cls, d: Dict[str, Any]) -> AppResourceApp: """Deserializes the AppResourceApp from a dictionary.""" - return cls() + return cls(name=d.get("name", None), permission=_enum(d, "permission", AppResourceAppAppPermission)) + + +class AppResourceAppAppPermission(Enum): + + CAN_USE = "CAN_USE" @dataclass @@ -1578,6 +1595,33 @@ class AppResourceUcSecurableUcSecurableType(Enum): VOLUME = "VOLUME" +@dataclass +class AppThumbnail: + """The thumbnail for an app.""" + + thumbnail: Optional[str] = None + """The thumbnail image bytes.""" + + def as_dict(self) -> dict: + """Serializes the AppThumbnail into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.thumbnail is not None: + body["thumbnail"] = self.thumbnail + return body + + def as_shallow_dict(self) -> dict: + """Serializes the AppThumbnail into a shallow dictionary of its immediate attributes.""" + body = {} + if self.thumbnail is not None: + body["thumbnail"] = self.thumbnail + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> AppThumbnail: + """Deserializes the AppThumbnail from a dictionary.""" + return cls(thumbnail=d.get("thumbnail", None)) + + @dataclass class AppUpdate: budget_policy_id: Optional[str] = None @@ -2984,6 +3028,25 @@ def delete(self, name: str) -> App: res = self._api.do("DELETE", f"/api/2.0/apps/{name}", headers=headers) return App.from_dict(res) + def delete_app_thumbnail(self, name: str): + """Deletes the thumbnail for an app. + + :param name: str + The name of the app. + + + """ + + headers = { + "Accept": "application/json", + } + + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id + + self._api.do("DELETE", f"/api/2.0/apps/{name}/thumbnail", headers=headers) + def delete_space(self, name: str) -> DeleteSpaceOperation: """Deletes an app space. @@ -3396,6 +3459,32 @@ def update(self, name: str, app: App) -> App: res = self._api.do("PATCH", f"/api/2.0/apps/{name}", body=body, headers=headers) return App.from_dict(res) + def update_app_thumbnail(self, name: str, *, app_thumbnail: Optional[AppThumbnail] = None) -> AppThumbnail: + """Updates the thumbnail for an app. + + :param name: str + The name of the app. + :param app_thumbnail: :class:`AppThumbnail` (optional) + The app thumbnail to set. + + :returns: :class:`AppThumbnail` + """ + + body = {} + if app_thumbnail is not None: + body["app_thumbnail"] = app_thumbnail.as_dict() + headers = { + "Accept": "application/json", + "Content-Type": "application/json", + } + + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id + + res = self._api.do("PATCH", f"/api/2.0/apps/{name}/thumbnail", body=body, headers=headers) + return AppThumbnail.from_dict(res) + def update_permissions( self, app_name: str, *, access_control_list: Optional[List[AppAccessControlRequest]] = None ) -> AppPermissions: diff --git a/databricks/sdk/service/catalog.py b/databricks/sdk/service/catalog.py index a8b75c73b..0826b6286 100755 --- a/databricks/sdk/service/catalog.py +++ b/databricks/sdk/service/catalog.py @@ -747,6 +747,46 @@ def from_dict(cls, d: Dict[str, Any]) -> AzureActiveDirectoryToken: return cls(aad_token=d.get("aad_token", None)) +@dataclass +class AzureEncryptionSettings: + azure_tenant_id: str + + azure_cmk_access_connector_id: Optional[str] = None + + azure_cmk_managed_identity_id: Optional[str] = None + + def as_dict(self) -> dict: + """Serializes the AzureEncryptionSettings into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.azure_cmk_access_connector_id is not None: + body["azure_cmk_access_connector_id"] = self.azure_cmk_access_connector_id + if self.azure_cmk_managed_identity_id is not None: + body["azure_cmk_managed_identity_id"] = self.azure_cmk_managed_identity_id + if self.azure_tenant_id is not None: + body["azure_tenant_id"] = self.azure_tenant_id + return body + + def as_shallow_dict(self) -> dict: + """Serializes the AzureEncryptionSettings into a shallow dictionary of its immediate attributes.""" + body = {} + if self.azure_cmk_access_connector_id is not None: + body["azure_cmk_access_connector_id"] = self.azure_cmk_access_connector_id + if self.azure_cmk_managed_identity_id is not None: + body["azure_cmk_managed_identity_id"] = self.azure_cmk_managed_identity_id + if self.azure_tenant_id is not None: + body["azure_tenant_id"] = self.azure_tenant_id + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> AzureEncryptionSettings: + """Deserializes the AzureEncryptionSettings from a dictionary.""" + return cls( + azure_cmk_access_connector_id=d.get("azure_cmk_access_connector_id", None), + azure_cmk_managed_identity_id=d.get("azure_cmk_managed_identity_id", None), + azure_tenant_id=d.get("azure_tenant_id", None), + ) + + @dataclass class AzureManagedIdentity: """The Azure managed identity configuration.""" @@ -1090,6 +1130,9 @@ class CatalogInfo: isolation_mode: Optional[CatalogIsolationMode] = None """Whether the current securable is accessible from all workspaces or a specific set of workspaces.""" + managed_encryption_settings: Optional[EncryptionSettings] = None + """Control CMK encryption for managed catalog data""" + metastore_id: Optional[str] = None """Unique identifier of parent metastore.""" @@ -1152,6 +1195,8 @@ def as_dict(self) -> dict: body["full_name"] = self.full_name if self.isolation_mode is not None: body["isolation_mode"] = self.isolation_mode.value + if self.managed_encryption_settings: + body["managed_encryption_settings"] = self.managed_encryption_settings.as_dict() if self.metastore_id is not None: body["metastore_id"] = self.metastore_id if self.name is not None: @@ -1203,6 +1248,8 @@ def as_shallow_dict(self) -> dict: body["full_name"] = self.full_name if self.isolation_mode is not None: body["isolation_mode"] = self.isolation_mode + if self.managed_encryption_settings: + body["managed_encryption_settings"] = self.managed_encryption_settings if self.metastore_id is not None: body["metastore_id"] = self.metastore_id if self.name is not None: @@ -1247,6 +1294,7 @@ def from_dict(cls, d: Dict[str, Any]) -> CatalogInfo: enable_predictive_optimization=_enum(d, "enable_predictive_optimization", EnablePredictiveOptimization), full_name=d.get("full_name", None), isolation_mode=_enum(d, "isolation_mode", CatalogIsolationMode), + managed_encryption_settings=_from_dict(d, "managed_encryption_settings", EncryptionSettings), metastore_id=d.get("metastore_id", None), name=d.get("name", None), options=d.get("options", None), @@ -1622,7 +1670,7 @@ def from_dict(cls, d: Dict[str, Any]) -> ConnectionDependency: @dataclass class ConnectionInfo: - """Next ID: 24""" + """Next ID: 25""" comment: Optional[str] = None """User-provided free-form text description.""" @@ -1784,7 +1832,7 @@ def from_dict(cls, d: Dict[str, Any]) -> ConnectionInfo: class ConnectionType(Enum): - """Next Id: 72""" + """Next Id: 75""" BIGQUERY = "BIGQUERY" DATABRICKS = "DATABRICKS" @@ -2597,7 +2645,7 @@ class CredentialPurpose(Enum): class CredentialType(Enum): - """Next Id: 17""" + """Next Id: 18""" ANY_STATIC_CREDENTIAL = "ANY_STATIC_CREDENTIAL" BEARER_TOKEN = "BEARER_TOKEN" @@ -3270,6 +3318,52 @@ def from_dict(cls, d: Dict[str, Any]) -> EncryptionDetails: return cls(sse_encryption_details=_from_dict(d, "sse_encryption_details", SseEncryptionDetails)) +@dataclass +class EncryptionSettings: + """Encryption Settings are used to carry metadata for securable encryption at rest. Currently used + for catalogs, we can use the information supplied here to interact with a CMK.""" + + azure_encryption_settings: Optional[AzureEncryptionSettings] = None + """optional Azure settings - only required if an Azure CMK is used.""" + + azure_key_vault_key_id: Optional[str] = None + """the AKV URL in Azure, null otherwise.""" + + customer_managed_key_id: Optional[str] = None + """the CMK uuid in AWS and GCP, null otherwise.""" + + def as_dict(self) -> dict: + """Serializes the EncryptionSettings into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.azure_encryption_settings: + body["azure_encryption_settings"] = self.azure_encryption_settings.as_dict() + if self.azure_key_vault_key_id is not None: + body["azure_key_vault_key_id"] = self.azure_key_vault_key_id + if self.customer_managed_key_id is not None: + body["customer_managed_key_id"] = self.customer_managed_key_id + return body + + def as_shallow_dict(self) -> dict: + """Serializes the EncryptionSettings into a shallow dictionary of its immediate attributes.""" + body = {} + if self.azure_encryption_settings: + body["azure_encryption_settings"] = self.azure_encryption_settings + if self.azure_key_vault_key_id is not None: + body["azure_key_vault_key_id"] = self.azure_key_vault_key_id + if self.customer_managed_key_id is not None: + body["customer_managed_key_id"] = self.customer_managed_key_id + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> EncryptionSettings: + """Deserializes the EncryptionSettings from a dictionary.""" + return cls( + azure_encryption_settings=_from_dict(d, "azure_encryption_settings", AzureEncryptionSettings), + azure_key_vault_key_id=d.get("azure_key_vault_key_id", None), + customer_managed_key_id=d.get("customer_managed_key_id", None), + ) + + @dataclass class EntityTagAssignment: """Represents a tag assignment to an entity""" @@ -3281,8 +3375,7 @@ class EntityTagAssignment: """The key of the tag""" entity_type: str - """The type of the entity to which the tag is assigned. Allowed values are: catalogs, schemas, - tables, columns, volumes.""" + """The type of the entity to which the tag is assigned.""" source_type: Optional[TagAssignmentSourceType] = None """The source type of the tag assignment, e.g., user-assigned or system-assigned""" @@ -8881,7 +8974,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Securable: class SecurableKind(Enum): - """Latest kind: CONNECTION_GOOGLE_DRIVE_SERVICE_ACCOUNT = 301; Next id: 302""" + """Latest kind: CONNECTION_VEEVA_VAULT_OAUTH_M2M = 311; Next id: 312""" TABLE_DB_STORAGE = "TABLE_DB_STORAGE" TABLE_DELTA = "TABLE_DELTA" @@ -8893,6 +8986,7 @@ class SecurableKind(Enum): TABLE_DELTA_ICEBERG_MANAGED = "TABLE_DELTA_ICEBERG_MANAGED" TABLE_DELTA_UNIFORM_HUDI_EXTERNAL = "TABLE_DELTA_UNIFORM_HUDI_EXTERNAL" TABLE_DELTA_UNIFORM_ICEBERG_EXTERNAL = "TABLE_DELTA_UNIFORM_ICEBERG_EXTERNAL" + TABLE_DELTA_UNIFORM_ICEBERG_EXTERNAL_DELTASHARING = "TABLE_DELTA_UNIFORM_ICEBERG_EXTERNAL_DELTASHARING" TABLE_DELTA_UNIFORM_ICEBERG_FOREIGN_DELTASHARING = "TABLE_DELTA_UNIFORM_ICEBERG_FOREIGN_DELTASHARING" TABLE_DELTA_UNIFORM_ICEBERG_FOREIGN_HIVE_METASTORE_EXTERNAL = ( "TABLE_DELTA_UNIFORM_ICEBERG_FOREIGN_HIVE_METASTORE_EXTERNAL" @@ -11229,6 +11323,7 @@ def create( *, comment: Optional[str] = None, connection_name: Optional[str] = None, + managed_encryption_settings: Optional[EncryptionSettings] = None, options: Optional[Dict[str, str]] = None, properties: Optional[Dict[str, str]] = None, provider_name: Optional[str] = None, @@ -11244,6 +11339,8 @@ def create( User-provided free-form text description. :param connection_name: str (optional) The name of the connection to an external data source. + :param managed_encryption_settings: :class:`EncryptionSettings` (optional) + Control CMK encryption for managed catalog data :param options: Dict[str,str] (optional) A map of key-value properties attached to the securable. :param properties: Dict[str,str] (optional) @@ -11265,6 +11362,8 @@ def create( body["comment"] = comment if connection_name is not None: body["connection_name"] = connection_name + if managed_encryption_settings is not None: + body["managed_encryption_settings"] = managed_encryption_settings.as_dict() if name is not None: body["name"] = name if options is not None: @@ -11416,6 +11515,7 @@ def update( comment: Optional[str] = None, enable_predictive_optimization: Optional[EnablePredictiveOptimization] = None, isolation_mode: Optional[CatalogIsolationMode] = None, + managed_encryption_settings: Optional[EncryptionSettings] = None, new_name: Optional[str] = None, options: Optional[Dict[str, str]] = None, owner: Optional[str] = None, @@ -11432,6 +11532,8 @@ def update( Whether predictive optimization should be enabled for this object and objects under it. :param isolation_mode: :class:`CatalogIsolationMode` (optional) Whether the current securable is accessible from all workspaces or a specific set of workspaces. + :param managed_encryption_settings: :class:`EncryptionSettings` (optional) + Control CMK encryption for managed catalog data :param new_name: str (optional) New name for the catalog. :param options: Dict[str,str] (optional) @@ -11451,6 +11553,8 @@ def update( body["enable_predictive_optimization"] = enable_predictive_optimization.value if isolation_mode is not None: body["isolation_mode"] = isolation_mode.value + if managed_encryption_settings is not None: + body["managed_encryption_settings"] = managed_encryption_settings.as_dict() if new_name is not None: body["new_name"] = new_name if options is not None: @@ -12105,8 +12209,7 @@ def delete(self, entity_type: str, entity_name: str, tag_key: str): [Manage tag policy permissions]: https://docs.databricks.com/aws/en/admin/tag-policies/manage-permissions :param entity_type: str - The type of the entity to which the tag is assigned. Allowed values are: catalogs, schemas, tables, - columns, volumes. + The type of the entity to which the tag is assigned. :param entity_name: str The fully qualified name of the entity to which the tag is assigned :param tag_key: str @@ -12133,8 +12236,7 @@ def get(self, entity_type: str, entity_name: str, tag_key: str) -> EntityTagAssi """Gets a tag assignment for an Unity Catalog entity by tag key. :param entity_type: str - The type of the entity to which the tag is assigned. Allowed values are: catalogs, schemas, tables, - columns, volumes. + The type of the entity to which the tag is assigned. :param entity_name: str The fully qualified name of the entity to which the tag is assigned :param tag_key: str @@ -12168,8 +12270,7 @@ def list( which is the only indication that the end of results has been reached. :param entity_type: str - The type of the entity to which the tag is assigned. Allowed values are: catalogs, schemas, tables, - columns, volumes. + The type of the entity to which the tag is assigned. :param entity_name: str The fully qualified name of the entity to which the tag is assigned :param max_results: int (optional) @@ -12222,8 +12323,7 @@ def update( [Manage tag policy permissions]: https://docs.databricks.com/aws/en/admin/tag-policies/manage-permissions :param entity_type: str - The type of the entity to which the tag is assigned. Allowed values are: catalogs, schemas, tables, - columns, volumes. + The type of the entity to which the tag is assigned. :param entity_name: str The fully qualified name of the entity to which the tag is assigned :param tag_key: str diff --git a/databricks/sdk/service/compute.py b/databricks/sdk/service/compute.py index 5f28647bc..98e1c7436 100755 --- a/databricks/sdk/service/compute.py +++ b/databricks/sdk/service/compute.py @@ -3169,12 +3169,13 @@ class Environment: environment spec, only pip and java dependencies are supported.""" base_environment: Optional[str] = None - """The `base_environment` key refers to an `env.yaml` file that specifies an environment version - and a collection of dependencies required for the environment setup. This `env.yaml` file may - itself include a `base_environment` reference pointing to another `env_1.yaml` file. However, - when used as a base environment, `env_1.yaml` (or further nested references) will not be - processed or included in the final environment, meaning that the resolution of - `base_environment` references is not recursive.""" + """The base environment this environment is built on top of. A base environment defines the + environment version and a list of dependencies for serverless compute. The value can be a file + path to a custom `env.yaml` file (e.g., `/Workspace/path/to/env.yaml`). Support for a + Databricks-provided base environment ID (e.g., `workspace-base-environments/databricks_ai_v4`) + and workspace base environment ID (e.g., + `workspace-base-environments/dbe_b849b66e-b31a-4cb5-b161-1f2b10877fb7`) is in Beta. Either + `environment_version` or `base_environment` can be provided. For more information, see""" client: Optional[str] = None """Use `environment_version` instead.""" diff --git a/databricks/sdk/service/dashboards.py b/databricks/sdk/service/dashboards.py index 4ce2f0d33..9ce26afc2 100755 --- a/databricks/sdk/service/dashboards.py +++ b/databricks/sdk/service/dashboards.py @@ -587,7 +587,46 @@ class GenieEvalResultDetails: """Assessment of the evaluation result: good, bad, or needs review""" assessment_reasons: Optional[List[ScoreReason]] = None - """Reasons for the assessment score.""" + """Reasons for the assessment score. + + Assessment reasons describe why a Genie response was scored as BAD. + + Deterministic values (compared against the ground truth result): - EMPTY_RESULT: Genie's + generated SQL results were empty for this benchmark question. - RESULT_MISSING_ROWS: Genie's + generated SQL response is missing rows from the provided ground truth SQL. - RESULT_EXTRA_ROWS: + Genie's generated SQL response has more rows than the provided ground truth SQL. - + RESULT_MISSING_COLUMNS: Genie's generated SQL response is missing columns from the provided + ground truth SQL. - RESULT_EXTRA_COLUMNS: Genie's generated SQL response has more columns than + the provided ground truth SQL. - SINGLE_CELL_DIFFERENCE: Single value result was produced but + differs from ground truth result. - EMPTY_GOOD_SQL: The benchmark SQL returned an empty result. + - COLUMN_TYPE_DIFFERENCE: The values between the results match but the column type is different. + + LLM judge ratings explain the factors driving BAD results: - + LLM_JUDGE_MISSING_OR_INCORRECT_FILTER: Genie's generated SQL is missing a WHERE clause condition + or has incorrect filter logic that excludes/includes wrong data. - + LLM_JUDGE_INCOMPLETE_OR_PARTIAL_OUTPUT: Genie's generated SQL returns only some of the requested + data or columns, missing parts of what the ground truth SQL returns. - + LLM_JUDGE_MISINTERPRETATION_OF_USER_REQUEST: Genie's generated SQL fundamentally misunderstands + what the user is asking for, addressing the wrong question or goal. - + LLM_JUDGE_INSTRUCTION_COMPLIANCE_OR_MISSING_BUSINESS_LOGIC: Genie's generated SQL fails to apply + specified instructions or business logic that should be followed. - + LLM_JUDGE_INCORRECT_METRIC_CALCULATION: Genie's generated SQL uses incorrect logic or makes + wrong assumptions when calculating metrics. - LLM_JUDGE_INCORRECT_TABLE_OR_FIELD_USAGE: Genie's + generated SQL references wrong tables, columns, or uses fields that don't match the ground truth + SQL's intent. - LLM_JUDGE_INCORRECT_FUNCTION_USAGE: Genie's generated SQL uses SQL functions + incorrectly or inappropriately (wrong parameters, wrong function for the task, etc.). - + LLM_JUDGE_MISSING_OR_INCORRECT_JOIN: Genie's generated SQL is missing necessary joins between + tables or has incorrect join conditions/types that produce wrong results. - + LLM_JUDGE_MISSING_OR_INCORRECT_AGGREGATION: Genie's generated SQL is missing GROUP BY clauses or + has incorrect grouping that doesn't match the requested aggregation level. - + LLM_JUDGE_FORMATTING_ERROR: Genie's generated SQL output has incorrect formatting, ordering + (ORDER BY), or presentation issues that don't match expectations. - LLM_JUDGE_OTHER: LLM judge + identified an error that doesn't fall into other categories. + + Deprecated LLM judge values (kept for backward compatibility, do not use): - + LLM_JUDGE_MISSING_JOIN (deprecated) - LLM_JUDGE_WRONG_FILTER (deprecated) - + LLM_JUDGE_WRONG_AGGREGATION (deprecated) - LLM_JUDGE_WRONG_COLUMNS (deprecated) - + LLM_JUDGE_SYNTAX_ERROR (deprecated) - LLM_JUDGE_SEMANTIC_ERROR (deprecated)""" eval_run_status: Optional[EvaluationStatusType] = None """Current status of the evaluation run.""" @@ -755,12 +794,17 @@ def from_dict(cls, d: Dict[str, Any]) -> GenieEvalRunResponse: class GenieFeedback: """Feedback containing rating and optional comment""" + comment: Optional[str] = None + """Optional feedback comment text""" + rating: Optional[GenieFeedbackRating] = None """The feedback rating""" def as_dict(self) -> dict: """Serializes the GenieFeedback into a dictionary suitable for use as a JSON request body.""" body = {} + if self.comment is not None: + body["comment"] = self.comment if self.rating is not None: body["rating"] = self.rating.value return body @@ -768,6 +812,8 @@ def as_dict(self) -> dict: def as_shallow_dict(self) -> dict: """Serializes the GenieFeedback into a shallow dictionary of its immediate attributes.""" body = {} + if self.comment is not None: + body["comment"] = self.comment if self.rating is not None: body["rating"] = self.rating return body @@ -775,7 +821,7 @@ def as_shallow_dict(self) -> dict: @classmethod def from_dict(cls, d: Dict[str, Any]) -> GenieFeedback: """Deserializes the GenieFeedback from a dictionary.""" - return cls(rating=_enum(d, "rating", GenieFeedbackRating)) + return cls(comment=d.get("comment", None), rating=_enum(d, "rating", GenieFeedbackRating)) class GenieFeedbackRating(Enum): @@ -870,6 +916,40 @@ def from_dict(cls, d: Dict[str, Any]) -> GenieGetMessageQueryResultResponse: return cls(statement_response=_from_dict(d, "statement_response", sql.StatementResponse)) +@dataclass +class GenieListConversationCommentsResponse: + comments: Optional[List[GenieMessageComment]] = None + """List of comments in the conversation.""" + + next_page_token: Optional[str] = None + """Token to get the next page of results.""" + + def as_dict(self) -> dict: + """Serializes the GenieListConversationCommentsResponse into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.comments: + body["comments"] = [v.as_dict() for v in self.comments] + if self.next_page_token is not None: + body["next_page_token"] = self.next_page_token + return body + + def as_shallow_dict(self) -> dict: + """Serializes the GenieListConversationCommentsResponse into a shallow dictionary of its immediate attributes.""" + body = {} + if self.comments: + body["comments"] = self.comments + if self.next_page_token is not None: + body["next_page_token"] = self.next_page_token + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> GenieListConversationCommentsResponse: + """Deserializes the GenieListConversationCommentsResponse from a dictionary.""" + return cls( + comments=_repeated_dict(d, "comments", GenieMessageComment), next_page_token=d.get("next_page_token", None) + ) + + @dataclass class GenieListConversationMessagesResponse: messages: Optional[List[GenieMessage]] = None @@ -1007,6 +1087,40 @@ def from_dict(cls, d: Dict[str, Any]) -> GenieListEvalRunsResponse: ) +@dataclass +class GenieListMessageCommentsResponse: + comments: Optional[List[GenieMessageComment]] = None + """List of comments on the message.""" + + next_page_token: Optional[str] = None + """Token to get the next page of results.""" + + def as_dict(self) -> dict: + """Serializes the GenieListMessageCommentsResponse into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.comments: + body["comments"] = [v.as_dict() for v in self.comments] + if self.next_page_token is not None: + body["next_page_token"] = self.next_page_token + return body + + def as_shallow_dict(self) -> dict: + """Serializes the GenieListMessageCommentsResponse into a shallow dictionary of its immediate attributes.""" + body = {} + if self.comments: + body["comments"] = self.comments + if self.next_page_token is not None: + body["next_page_token"] = self.next_page_token + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> GenieListMessageCommentsResponse: + """Deserializes the GenieListMessageCommentsResponse from a dictionary.""" + return cls( + comments=_repeated_dict(d, "comments", GenieMessageComment), next_page_token=d.get("next_page_token", None) + ) + + @dataclass class GenieListSpacesResponse: next_page_token: Optional[str] = None @@ -1162,6 +1276,83 @@ def from_dict(cls, d: Dict[str, Any]) -> GenieMessage: ) +@dataclass +class GenieMessageComment: + """A comment on a Genie conversation message.""" + + space_id: str + """Genie space ID""" + + conversation_id: str + """Conversation ID""" + + message_id: str + """Message ID""" + + message_comment_id: str + """Comment ID""" + + content: str + """Comment text content""" + + created_timestamp: Optional[int] = None + """Timestamp when the comment was created""" + + user_id: Optional[int] = None + """ID of the user who created the comment""" + + def as_dict(self) -> dict: + """Serializes the GenieMessageComment into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.content is not None: + body["content"] = self.content + if self.conversation_id is not None: + body["conversation_id"] = self.conversation_id + if self.created_timestamp is not None: + body["created_timestamp"] = self.created_timestamp + if self.message_comment_id is not None: + body["message_comment_id"] = self.message_comment_id + if self.message_id is not None: + body["message_id"] = self.message_id + if self.space_id is not None: + body["space_id"] = self.space_id + if self.user_id is not None: + body["user_id"] = self.user_id + return body + + def as_shallow_dict(self) -> dict: + """Serializes the GenieMessageComment into a shallow dictionary of its immediate attributes.""" + body = {} + if self.content is not None: + body["content"] = self.content + if self.conversation_id is not None: + body["conversation_id"] = self.conversation_id + if self.created_timestamp is not None: + body["created_timestamp"] = self.created_timestamp + if self.message_comment_id is not None: + body["message_comment_id"] = self.message_comment_id + if self.message_id is not None: + body["message_id"] = self.message_id + if self.space_id is not None: + body["space_id"] = self.space_id + if self.user_id is not None: + body["user_id"] = self.user_id + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> GenieMessageComment: + """Deserializes the GenieMessageComment from a dictionary.""" + return cls( + content=d.get("content", None), + conversation_id=d.get("conversation_id", None), + created_timestamp=d.get("created_timestamp", None), + message_comment_id=d.get("message_comment_id", None), + message_id=d.get("message_id", None), + space_id=d.get("space_id", None), + user_id=d.get("user_id", None), + ) + + @dataclass class GenieQueryAttachment: description: Optional[str] = None @@ -1184,6 +1375,9 @@ class GenieQueryAttachment: """Statement Execution API statement id. Use [Get status, manifest, and result first chunk](:method:statementexecution/getstatement) to get the full result data.""" + thoughts: Optional[List[Thought]] = None + """Insights into how Genie came to generate the SQL.""" + title: Optional[str] = None """Name of the query""" @@ -1204,6 +1398,8 @@ def as_dict(self) -> dict: body["query_result_metadata"] = self.query_result_metadata.as_dict() if self.statement_id is not None: body["statement_id"] = self.statement_id + if self.thoughts: + body["thoughts"] = [v.as_dict() for v in self.thoughts] if self.title is not None: body["title"] = self.title return body @@ -1225,6 +1421,8 @@ def as_shallow_dict(self) -> dict: body["query_result_metadata"] = self.query_result_metadata if self.statement_id is not None: body["statement_id"] = self.statement_id + if self.thoughts: + body["thoughts"] = self.thoughts if self.title is not None: body["title"] = self.title return body @@ -1240,6 +1438,7 @@ def from_dict(cls, d: Dict[str, Any]) -> GenieQueryAttachment: query=d.get("query", None), query_result_metadata=_from_dict(d, "query_result_metadata", GenieResultMetadata), statement_id=d.get("statement_id", None), + thoughts=_repeated_dict(d, "thoughts", Thought), title=d.get("title", None), ) @@ -2194,6 +2393,56 @@ class TextAttachmentPurpose(Enum): FOLLOW_UP_QUESTION = "FOLLOW_UP_QUESTION" +@dataclass +class Thought: + """A single thought in the AI's reasoning process for a query.""" + + content: Optional[str] = None + """The md formatted content for this thought.""" + + thought_type: Optional[ThoughtType] = None + """The category of this thought.""" + + def as_dict(self) -> dict: + """Serializes the Thought into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.content is not None: + body["content"] = self.content + if self.thought_type is not None: + body["thought_type"] = self.thought_type.value + return body + + def as_shallow_dict(self) -> dict: + """Serializes the Thought into a shallow dictionary of its immediate attributes.""" + body = {} + if self.content is not None: + body["content"] = self.content + if self.thought_type is not None: + body["thought_type"] = self.thought_type + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> Thought: + """Deserializes the Thought from a dictionary.""" + return cls(content=d.get("content", None), thought_type=_enum(d, "thought_type", ThoughtType)) + + +class ThoughtType(Enum): + """ThoughtType. The possible values are: * `THOUGHT_TYPE_UNSPECIFIED`: Default value that should + not be used. * `THOUGHT_TYPE_DESCRIPTION`: A high-level description of how the question was + interpreted. * `THOUGHT_TYPE_UNDERSTANDING`: How ambiguous parts of the question were resolved. + * `THOUGHT_TYPE_DATA_SOURCING`: Which tables or datasets were identified as relevant. * + `THOUGHT_TYPE_INSTRUCTIONS`: Which author-defined instructions were referenced. * + `THOUGHT_TYPE_STEPS`: The logical steps taken to compute the answer. The category of a Thought. + Additional values may be added in the future.""" + + THOUGHT_TYPE_DATA_SOURCING = "THOUGHT_TYPE_DATA_SOURCING" + THOUGHT_TYPE_DESCRIPTION = "THOUGHT_TYPE_DESCRIPTION" + THOUGHT_TYPE_INSTRUCTIONS = "THOUGHT_TYPE_INSTRUCTIONS" + THOUGHT_TYPE_STEPS = "THOUGHT_TYPE_STEPS" + THOUGHT_TYPE_UNDERSTANDING = "THOUGHT_TYPE_UNDERSTANDING" + + @dataclass class TrashDashboardResponse: def as_dict(self) -> dict: @@ -2322,6 +2571,43 @@ def create_message_and_wait( timeout=timeout ) + def create_message_comment( + self, space_id: str, conversation_id: str, message_id: str, content: str + ) -> GenieMessageComment: + """Create a comment on a conversation message. + + :param space_id: str + The ID associated with the Genie space. + :param conversation_id: str + The ID associated with the conversation. + :param message_id: str + The ID associated with the message. + :param content: str + Comment text content. + + :returns: :class:`GenieMessageComment` + """ + + body = {} + if content is not None: + body["content"] = content + headers = { + "Accept": "application/json", + "Content-Type": "application/json", + } + + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id + + res = self._api.do( + "POST", + f"/api/2.0/genie/spaces/{space_id}/conversations/{conversation_id}/messages/{message_id}/comments", + body=body, + headers=headers, + ) + return GenieMessageComment.from_dict(res) + def create_space( self, warehouse_id: str, @@ -2896,6 +3182,44 @@ def get_space(self, space_id: str, *, include_serialized_space: Optional[bool] = res = self._api.do("GET", f"/api/2.0/genie/spaces/{space_id}", query=query, headers=headers) return GenieSpace.from_dict(res) + def list_conversation_comments( + self, space_id: str, conversation_id: str, *, page_size: Optional[int] = None, page_token: Optional[str] = None + ) -> GenieListConversationCommentsResponse: + """List all comments across all messages in a conversation. + + :param space_id: str + The ID associated with the Genie space. + :param conversation_id: str + The ID associated with the conversation. + :param page_size: int (optional) + Maximum number of comments to return per page. + :param page_token: str (optional) + Pagination token for getting the next page of results. + + :returns: :class:`GenieListConversationCommentsResponse` + """ + + query = {} + if page_size is not None: + query["page_size"] = page_size + if page_token is not None: + query["page_token"] = page_token + headers = { + "Accept": "application/json", + } + + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id + + res = self._api.do( + "GET", + f"/api/2.0/genie/spaces/{space_id}/conversations/{conversation_id}/list-comments", + query=query, + headers=headers, + ) + return GenieListConversationCommentsResponse.from_dict(res) + def list_conversation_messages( self, space_id: str, conversation_id: str, *, page_size: Optional[int] = None, page_token: Optional[str] = None ) -> GenieListConversationMessagesResponse: @@ -2975,6 +3299,52 @@ def list_conversations( res = self._api.do("GET", f"/api/2.0/genie/spaces/{space_id}/conversations", query=query, headers=headers) return GenieListConversationsResponse.from_dict(res) + def list_message_comments( + self, + space_id: str, + conversation_id: str, + message_id: str, + *, + page_size: Optional[int] = None, + page_token: Optional[str] = None, + ) -> GenieListMessageCommentsResponse: + """List comments on a specific conversation message. + + :param space_id: str + The ID associated with the Genie space. + :param conversation_id: str + The ID associated with the conversation. + :param message_id: str + The ID associated with the message. + :param page_size: int (optional) + Maximum number of comments to return per page. + :param page_token: str (optional) + Pagination token for getting the next page of results. + + :returns: :class:`GenieListMessageCommentsResponse` + """ + + query = {} + if page_size is not None: + query["page_size"] = page_size + if page_token is not None: + query["page_token"] = page_token + headers = { + "Accept": "application/json", + } + + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id + + res = self._api.do( + "GET", + f"/api/2.0/genie/spaces/{space_id}/conversations/{conversation_id}/messages/{message_id}/comments", + query=query, + headers=headers, + ) + return GenieListMessageCommentsResponse.from_dict(res) + def list_spaces( self, *, page_size: Optional[int] = None, page_token: Optional[str] = None ) -> GenieListSpacesResponse: @@ -3004,7 +3374,15 @@ def list_spaces( res = self._api.do("GET", "/api/2.0/genie/spaces", query=query, headers=headers) return GenieListSpacesResponse.from_dict(res) - def send_message_feedback(self, space_id: str, conversation_id: str, message_id: str, rating: GenieFeedbackRating): + def send_message_feedback( + self, + space_id: str, + conversation_id: str, + message_id: str, + rating: GenieFeedbackRating, + *, + comment: Optional[str] = None, + ): """Send feedback for a message. :param space_id: str @@ -3015,11 +3393,15 @@ def send_message_feedback(self, space_id: str, conversation_id: str, message_id: The ID associated with the message to provide feedback for. :param rating: :class:`GenieFeedbackRating` The rating (POSITIVE, NEGATIVE, or NONE). + :param comment: str (optional) + Optional text feedback that will be stored as a comment. """ body = {} + if comment is not None: + body["comment"] = comment if rating is not None: body["rating"] = rating.value headers = { diff --git a/databricks/sdk/service/iam.py b/databricks/sdk/service/iam.py index 17afb4a50..780587698 100755 --- a/databricks/sdk/service/iam.py +++ b/databricks/sdk/service/iam.py @@ -5540,6 +5540,9 @@ def create( "Accept": "application/json", "Content-Type": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id res = self._api.do("POST", "/api/2.0/preview/scim/v2/Groups", body=body, headers=headers) return Group.from_dict(res) @@ -5554,6 +5557,9 @@ def delete(self, id: str): """ headers = {} + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id self._api.do("DELETE", f"/api/2.0/preview/scim/v2/Groups/{id}", headers=headers) @@ -5569,6 +5575,9 @@ def get(self, id: str) -> Group: headers = { "Accept": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id res = self._api.do("GET", f"/api/2.0/preview/scim/v2/Groups/{id}", headers=headers) return Group.from_dict(res) @@ -5627,6 +5636,9 @@ def list( headers = { "Accept": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id # deduplicate items that may have been added during iteration seen = set() @@ -5665,6 +5677,9 @@ def patch(self, id: str, *, operations: Optional[List[Patch]] = None, schemas: O headers = { "Content-Type": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id self._api.do("PATCH", f"/api/2.0/preview/scim/v2/Groups/{id}", body=body, headers=headers) @@ -5724,6 +5739,9 @@ def update( headers = { "Content-Type": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id self._api.do("PUT", f"/api/2.0/preview/scim/v2/Groups/{id}", body=body, headers=headers) @@ -5798,6 +5816,9 @@ def create( "Accept": "application/json", "Content-Type": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id res = self._api.do("POST", "/api/2.0/preview/scim/v2/ServicePrincipals", body=body, headers=headers) return ServicePrincipal.from_dict(res) @@ -5812,6 +5833,9 @@ def delete(self, id: str): """ headers = {} + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id self._api.do("DELETE", f"/api/2.0/preview/scim/v2/ServicePrincipals/{id}", headers=headers) @@ -5827,6 +5851,9 @@ def get(self, id: str) -> ServicePrincipal: headers = { "Accept": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id res = self._api.do("GET", f"/api/2.0/preview/scim/v2/ServicePrincipals/{id}", headers=headers) return ServicePrincipal.from_dict(res) @@ -5885,6 +5912,9 @@ def list( headers = { "Accept": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id # deduplicate items that may have been added during iteration seen = set() @@ -5923,6 +5953,9 @@ def patch(self, id: str, *, operations: Optional[List[Patch]] = None, schemas: O headers = { "Content-Type": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id self._api.do("PATCH", f"/api/2.0/preview/scim/v2/ServicePrincipals/{id}", body=body, headers=headers) @@ -5985,6 +6018,9 @@ def update( headers = { "Content-Type": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id self._api.do("PUT", f"/api/2.0/preview/scim/v2/ServicePrincipals/{id}", body=body, headers=headers) @@ -6077,6 +6113,9 @@ def create( "Accept": "application/json", "Content-Type": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id res = self._api.do("POST", "/api/2.0/preview/scim/v2/Users", body=body, headers=headers) return User.from_dict(res) @@ -6092,6 +6131,9 @@ def delete(self, id: str): """ headers = {} + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id self._api.do("DELETE", f"/api/2.0/preview/scim/v2/Users/{id}", headers=headers) @@ -6153,6 +6195,9 @@ def get( headers = { "Accept": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id res = self._api.do("GET", f"/api/2.0/preview/scim/v2/Users/{id}", query=query, headers=headers) return User.from_dict(res) @@ -6167,6 +6212,9 @@ def get_permission_levels(self) -> GetPasswordPermissionLevelsResponse: headers = { "Accept": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id res = self._api.do("GET", "/api/2.0/permissions/authorization/passwords/permissionLevels", headers=headers) return GetPasswordPermissionLevelsResponse.from_dict(res) @@ -6181,6 +6229,9 @@ def get_permissions(self) -> PasswordPermissions: headers = { "Accept": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id res = self._api.do("GET", "/api/2.0/permissions/authorization/passwords", headers=headers) return PasswordPermissions.from_dict(res) @@ -6240,6 +6291,9 @@ def list( headers = { "Accept": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id # deduplicate items that may have been added during iteration seen = set() @@ -6278,6 +6332,9 @@ def patch(self, id: str, *, operations: Optional[List[Patch]] = None, schemas: O headers = { "Content-Type": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id self._api.do("PATCH", f"/api/2.0/preview/scim/v2/Users/{id}", body=body, headers=headers) @@ -6298,6 +6355,9 @@ def set_permissions( "Accept": "application/json", "Content-Type": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id res = self._api.do("PUT", "/api/2.0/permissions/authorization/passwords", body=body, headers=headers) return PasswordPermissions.from_dict(res) @@ -6372,6 +6432,9 @@ def update( headers = { "Content-Type": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id self._api.do("PUT", f"/api/2.0/preview/scim/v2/Users/{id}", body=body, headers=headers) @@ -6391,6 +6454,9 @@ def update_permissions( "Accept": "application/json", "Content-Type": "application/json", } + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id res = self._api.do("PATCH", "/api/2.0/permissions/authorization/passwords", body=body, headers=headers) return PasswordPermissions.from_dict(res) diff --git a/databricks/sdk/service/jobs.py b/databricks/sdk/service/jobs.py index 005591fd2..eda68ce0d 100755 --- a/databricks/sdk/service/jobs.py +++ b/databricks/sdk/service/jobs.py @@ -5842,7 +5842,8 @@ class RunTask: used to reference the tasks to be updated or reset.""" alert_task: Optional[AlertTask] = None - """New alert v2 task""" + """The task evaluates a Databricks alert and sends notifications to subscribers when the + `alert_task` field is present.""" attempt_number: Optional[int] = None """The sequence number of this run attempt for a triggered job run. The initial attempt of a run @@ -7167,7 +7168,8 @@ class SubmitTask: used to reference the tasks to be updated or reset.""" alert_task: Optional[AlertTask] = None - """New alert v2 task""" + """The task evaluates a Databricks alert and sends notifications to subscribers when the + `alert_task` field is present.""" clean_rooms_notebook_task: Optional[CleanRoomsNotebookTask] = None """The task runs a [clean rooms] notebook when the `clean_rooms_notebook_task` field is present. @@ -7699,7 +7701,8 @@ class Task: used to reference the tasks to be updated or reset.""" alert_task: Optional[AlertTask] = None - """New alert v2 task""" + """The task evaluates a Databricks alert and sends notifications to subscribers when the + `alert_task` field is present.""" clean_rooms_notebook_task: Optional[CleanRoomsNotebookTask] = None """The task runs a [clean rooms] notebook when the `clean_rooms_notebook_task` field is present. @@ -8254,7 +8257,9 @@ class TerminationCodeCode(Enum): `MAX_JOB_QUEUE_SIZE_EXCEEDED`: The run was skipped due to reaching the job level queue size limit. * `DISABLED`: The run was never executed because it was disabled explicitly by the user. * `BREAKING_CHANGE`: Run failed because of an intentional breaking change in Spark, but it will - be retried with a mitigation config. + be retried with a mitigation config. * `CLUSTER_TERMINATED_BY_USER`: The run failed because the + externally managed cluster entered an unusable state, likely due to the user terminating or + restarting it outside the jobs service. [Link]: https://kb.databricks.com/en_US/notebooks/too-many-execution-contexts-are-open-right-now""" diff --git a/databricks/sdk/service/ml.py b/databricks/sdk/service/ml.py index 6dad9e8d0..51a62f08a 100755 --- a/databricks/sdk/service/ml.py +++ b/databricks/sdk/service/ml.py @@ -428,7 +428,10 @@ class AvgFunction: """Computes the average of values.""" input: str - """The input column from which the average is computed.""" + """The input column from which the average is computed. For Kafka sources, use dot-prefixed path + notation (e.g., "value.amount"). For nested fields, the leaf node name is used. TODO(FS-939): + Colon-prefixed notation (e.g., "value:amount") is supported for backwards compatibility but is + deprecated; migrate to dot notation.""" def as_dict(self) -> dict: """Serializes the AvgFunction into a dictionary suitable for use as a JSON request body.""" @@ -505,9 +508,10 @@ def from_dict(cls, d: Dict[str, Any]) -> BatchCreateMaterializedFeaturesResponse @dataclass class ColumnIdentifier: variant_expr_path: str - """String representation of the column name or variant expression path. For nested fields, the leaf - value is what will be present in materialized tables and expected to match at query time. For - example, the leaf node of value:trip_details.location_details.pickup_zip is pickup_zip.""" + """String representation of the column name using dot-prefixed path notation. For nested fields, + the leaf value is what will be present in materialized tables and expected to match at query + time. For example, the leaf node of value.trip_details.location_details.pickup_zip is + pickup_zip.""" def as_dict(self) -> dict: """Serializes the ColumnIdentifier into a dictionary suitable for use as a JSON request body.""" @@ -684,7 +688,10 @@ class CountFunction: """Computes the count of values.""" input: str - """The input column from which the count is computed.""" + """The input column from which the count is computed. For Kafka sources, use dot-prefixed path + notation (e.g., "value.amount"). For nested fields, the leaf node name is used. TODO(FS-939): + Colon-prefixed notation (e.g., "value:amount") is supported for backwards compatibility but is + deprecated; migrate to dot notation.""" def as_dict(self) -> dict: """Serializes the CountFunction into a dictionary suitable for use as a JSON request body.""" @@ -956,9 +963,16 @@ def from_dict(cls, d: Dict[str, Any]) -> CreateWebhookResponse: @dataclass class DataSource: + """Specifies the data source backing a feature. Exactly one source type must be set.""" + delta_table_source: Optional[DeltaTableSource] = None + """A Delta table data source.""" kafka_source: Optional[KafkaSource] = None + """A Kafka stream data source.""" + + request_source: Optional[RequestSource] = None + """A request-time data source.""" def as_dict(self) -> dict: """Serializes the DataSource into a dictionary suitable for use as a JSON request body.""" @@ -967,6 +981,8 @@ def as_dict(self) -> dict: body["delta_table_source"] = self.delta_table_source.as_dict() if self.kafka_source: body["kafka_source"] = self.kafka_source.as_dict() + if self.request_source: + body["request_source"] = self.request_source.as_dict() return body def as_shallow_dict(self) -> dict: @@ -976,6 +992,8 @@ def as_shallow_dict(self) -> dict: body["delta_table_source"] = self.delta_table_source if self.kafka_source: body["kafka_source"] = self.kafka_source + if self.request_source: + body["request_source"] = self.request_source return body @classmethod @@ -984,6 +1002,7 @@ def from_dict(cls, d: Dict[str, Any]) -> DataSource: return cls( delta_table_source=_from_dict(d, "delta_table_source", DeltaTableSource), kafka_source=_from_dict(d, "kafka_source", KafkaSource), + request_source=_from_dict(d, "request_source", RequestSource), ) @@ -1421,7 +1440,12 @@ def from_dict(cls, d: Dict[str, Any]) -> DeltaTableSource: @dataclass class EntityColumn: name: str - """The name of the entity column.""" + """The name of the entity column. For Kafka sources, use dot-prefixed path notation to reference + fields within the key or value schema (e.g., "value.user_id", "key.partition_key"). For nested + fields, the leaf node name (e.g., "user_id" from "value.trip_details.user_id") is what will be + present in materialized tables and expected to match at query time. TODO(FS-939): Colon-prefixed + notation (e.g., "value:user_id") is supported for backwards compatibility but is deprecated; + migrate to dot notation.""" def as_dict(self) -> dict: """Serializes the EntityColumn into a dictionary suitable for use as a JSON request body.""" @@ -2081,6 +2105,41 @@ def from_dict(cls, d: Dict[str, Any]) -> FeatureTag: return cls(key=d.get("key", None), value=d.get("value", None)) +@dataclass +class FieldDefinition: + """A single field definition within a FlatSchema, specifying the field name and its scalar data + type. Does not support nested or complex types (arrays, maps, structs).""" + + name: str + """The name of the field.""" + + data_type: ScalarDataType + """The scalar data type of the field.""" + + def as_dict(self) -> dict: + """Serializes the FieldDefinition into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.data_type is not None: + body["data_type"] = self.data_type.value + if self.name is not None: + body["name"] = self.name + return body + + def as_shallow_dict(self) -> dict: + """Serializes the FieldDefinition into a shallow dictionary of its immediate attributes.""" + body = {} + if self.data_type is not None: + body["data_type"] = self.data_type + if self.name is not None: + body["name"] = self.name + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> FieldDefinition: + """Deserializes the FieldDefinition from a dictionary.""" + return cls(data_type=_enum(d, "data_type", ScalarDataType), name=d.get("name", None)) + + @dataclass class FileInfo: """Metadata of a single artifact file or directory.""" @@ -2174,6 +2233,34 @@ def from_dict(cls, d: Dict[str, Any]) -> FirstFunction: return cls(input=d.get("input", None)) +@dataclass +class FlatSchema: + """A flat (non-nested) schema for request-time fields, defined as an ordered list of field + definitions. This schema only supports scalar types.""" + + fields: List[FieldDefinition] + """The list of fields in this schema.""" + + def as_dict(self) -> dict: + """Serializes the FlatSchema into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.fields: + body["fields"] = [v.as_dict() for v in self.fields] + return body + + def as_shallow_dict(self) -> dict: + """Serializes the FlatSchema into a shallow dictionary of its immediate attributes.""" + body = {} + if self.fields: + body["fields"] = self.fields + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> FlatSchema: + """Deserializes the FlatSchema from a dictionary.""" + return cls(fields=_repeated_dict(d, "fields", FieldDefinition)) + + @dataclass class ForecastingExperiment: """Represents a forecasting experiment with its unique identifier, URL, and state.""" @@ -3854,6 +3941,9 @@ class MaterializedFeature: """The quartz cron expression that defines the schedule of the materialization pipeline. The schedule is evaluated in the UTC timezone.""" + is_online: Optional[bool] = None + """True if this is an online materialized feature. False if it is an offline materialized feature.""" + last_materialization_time: Optional[str] = None """The timestamp when the pipeline last ran and updated the materialized feature values. If the pipeline has not run yet, this field will be null.""" @@ -3879,6 +3969,8 @@ def as_dict(self) -> dict: body["cron_schedule"] = self.cron_schedule if self.feature_name is not None: body["feature_name"] = self.feature_name + if self.is_online is not None: + body["is_online"] = self.is_online if self.last_materialization_time is not None: body["last_materialization_time"] = self.last_materialization_time if self.materialized_feature_id is not None: @@ -3900,6 +3992,8 @@ def as_shallow_dict(self) -> dict: body["cron_schedule"] = self.cron_schedule if self.feature_name is not None: body["feature_name"] = self.feature_name + if self.is_online is not None: + body["is_online"] = self.is_online if self.last_materialization_time is not None: body["last_materialization_time"] = self.last_materialization_time if self.materialized_feature_id is not None: @@ -3920,6 +4014,7 @@ def from_dict(cls, d: Dict[str, Any]) -> MaterializedFeature: return cls( cron_schedule=d.get("cron_schedule", None), feature_name=d.get("feature_name", None), + is_online=d.get("is_online", None), last_materialization_time=d.get("last_materialization_time", None), materialized_feature_id=d.get("materialized_feature_id", None), offline_store_config=_from_dict(d, "offline_store_config", OfflineStoreConfig), @@ -5421,6 +5516,34 @@ def from_dict(cls, d: Dict[str, Any]) -> RenameModelResponse: return cls(registered_model=_from_dict(d, "registered_model", Model)) +@dataclass +class RequestSource: + """A request-time data source whose value is provided at inference time: offline batch scoring or + online serving endpoint""" + + flat_schema: Optional[FlatSchema] = None + """A flat schema with scalar-typed fields only.""" + + def as_dict(self) -> dict: + """Serializes the RequestSource into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.flat_schema: + body["flat_schema"] = self.flat_schema.as_dict() + return body + + def as_shallow_dict(self) -> dict: + """Serializes the RequestSource into a shallow dictionary of its immediate attributes.""" + body = {} + if self.flat_schema: + body["flat_schema"] = self.flat_schema + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> RequestSource: + """Deserializes the RequestSource from a dictionary.""" + return cls(flat_schema=_from_dict(d, "flat_schema", FlatSchema)) + + @dataclass class RestoreExperimentResponse: def as_dict(self) -> dict: @@ -5758,6 +5881,23 @@ def from_dict(cls, d: Dict[str, Any]) -> RunTag: return cls(key=d.get("key", None), value=d.get("value", None)) +class ScalarDataType(Enum): + """Scalar data types for request-time field definitions. Only flat (non-nested) types are + supported.""" + + BINARY = "BINARY" + BOOLEAN = "BOOLEAN" + DATE = "DATE" + DECIMAL = "DECIMAL" + DOUBLE = "DOUBLE" + FLOAT = "FLOAT" + INTEGER = "INTEGER" + LONG = "LONG" + SHORT = "SHORT" + STRING = "STRING" + TIMESTAMP = "TIMESTAMP" + + @dataclass class SchemaConfig: json_schema: Optional[str] = None @@ -6179,7 +6319,10 @@ class StddevPopFunction: """Computes the population standard deviation.""" input: str - """The input column from which the population standard deviation is computed.""" + """The input column from which the population standard deviation is computed. For Kafka sources, + use dot-prefixed path notation (e.g., "value.amount"). For nested fields, the leaf node name is + used. TODO(FS-939): Colon-prefixed notation (e.g., "value:amount") is supported for backwards + compatibility but is deprecated; migrate to dot notation.""" def as_dict(self) -> dict: """Serializes the StddevPopFunction into a dictionary suitable for use as a JSON request body.""" @@ -6278,7 +6421,10 @@ class SumFunction: """Computes the sum of values.""" input: str - """The input column from which the sum is computed.""" + """The input column from which the sum is computed. For Kafka sources, use dot-prefixed path + notation (e.g., "value.amount"). For nested fields, the leaf node name is used. TODO(FS-939): + Colon-prefixed notation (e.g., "value:amount") is supported for backwards compatibility but is + deprecated; migrate to dot notation.""" def as_dict(self) -> dict: """Serializes the SumFunction into a dictionary suitable for use as a JSON request body.""" @@ -6375,7 +6521,12 @@ def from_dict(cls, d: Dict[str, Any]) -> TimeWindow: @dataclass class TimeseriesColumn: name: str - """The name of the timeseries column.""" + """The name of the timeseries column. For Kafka sources, use dot-prefixed path notation to + reference fields within the key or value schema (e.g., "value.event_timestamp"). For nested + fields, the leaf node name (e.g., "event_timestamp" from "value.event_details.event_timestamp") + is what will be present in materialized tables and expected to match at query time. + TODO(FS-939): Colon-prefixed notation (e.g., "value:event_timestamp") is supported for backwards + compatibility but is deprecated; migrate to dot notation.""" def as_dict(self) -> dict: """Serializes the TimeseriesColumn into a dictionary suitable for use as a JSON request body.""" diff --git a/databricks/sdk/service/pipelines.py b/databricks/sdk/service/pipelines.py index 7e8508274..4ed07db08 100755 --- a/databricks/sdk/service/pipelines.py +++ b/databricks/sdk/service/pipelines.py @@ -22,6 +22,24 @@ # all definitions in this file are in alphabetical order +@dataclass +class ApplyEnvironmentRequestResponse: + def as_dict(self) -> dict: + """Serializes the ApplyEnvironmentRequestResponse into a dictionary suitable for use as a JSON request body.""" + body = {} + return body + + def as_shallow_dict(self) -> dict: + """Serializes the ApplyEnvironmentRequestResponse into a shallow dictionary of its immediate attributes.""" + body = {} + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> ApplyEnvironmentRequestResponse: + """Deserializes the ApplyEnvironmentRequestResponse from a dictionary.""" + return cls() + + @dataclass class AutoFullRefreshPolicy: """Policy for auto full refresh.""" @@ -116,6 +134,55 @@ def from_dict(cls, d: Dict[str, Any]) -> ConnectionParameters: return cls(source_catalog=d.get("source_catalog", None)) +@dataclass +class ConnectorOptions: + """Wrapper message for source-specific options to support multiple connector types""" + + gdrive_options: Optional[GoogleDriveOptions] = None + + google_ads_options: Optional[GoogleAdsOptions] = None + + sharepoint_options: Optional[SharepointOptions] = None + + tiktok_ads_options: Optional[TikTokAdsOptions] = None + + def as_dict(self) -> dict: + """Serializes the ConnectorOptions into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.gdrive_options: + body["gdrive_options"] = self.gdrive_options.as_dict() + if self.google_ads_options: + body["google_ads_options"] = self.google_ads_options.as_dict() + if self.sharepoint_options: + body["sharepoint_options"] = self.sharepoint_options.as_dict() + if self.tiktok_ads_options: + body["tiktok_ads_options"] = self.tiktok_ads_options.as_dict() + return body + + def as_shallow_dict(self) -> dict: + """Serializes the ConnectorOptions into a shallow dictionary of its immediate attributes.""" + body = {} + if self.gdrive_options: + body["gdrive_options"] = self.gdrive_options + if self.google_ads_options: + body["google_ads_options"] = self.google_ads_options + if self.sharepoint_options: + body["sharepoint_options"] = self.sharepoint_options + if self.tiktok_ads_options: + body["tiktok_ads_options"] = self.tiktok_ads_options + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> ConnectorOptions: + """Deserializes the ConnectorOptions from a dictionary.""" + return cls( + gdrive_options=_from_dict(d, "gdrive_options", GoogleDriveOptions), + google_ads_options=_from_dict(d, "google_ads_options", GoogleAdsOptions), + sharepoint_options=_from_dict(d, "sharepoint_options", SharepointOptions), + tiktok_ads_options=_from_dict(d, "tiktok_ads_options", TikTokAdsOptions), + ) + + class ConnectorType(Enum): """For certain database sources LakeFlow Connect offers both query based and cdc ingestion, ConnectorType can bse used to convey the type of ingestion. If connection_name is provided for @@ -408,6 +475,182 @@ def from_dict(cls, d: Dict[str, Any]) -> EventLogSpec: return cls(catalog=d.get("catalog", None), name=d.get("name", None), schema=d.get("schema", None)) +@dataclass +class FileFilter: + modified_after: Optional[str] = None + """Include files with modification times occurring after the specified time. Timestamp format: + YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00) Based on + https://spark.apache.org/docs/latest/sql-data-sources-generic-options.html#modification-time-path-filters""" + + modified_before: Optional[str] = None + """Include files with modification times occurring before the specified time. Timestamp format: + YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00) Based on + https://spark.apache.org/docs/latest/sql-data-sources-generic-options.html#modification-time-path-filters""" + + path_filter: Optional[str] = None + """Include files with file names matching the pattern Based on + https://spark.apache.org/docs/latest/sql-data-sources-generic-options.html#path-glob-filter""" + + def as_dict(self) -> dict: + """Serializes the FileFilter into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.modified_after is not None: + body["modified_after"] = self.modified_after + if self.modified_before is not None: + body["modified_before"] = self.modified_before + if self.path_filter is not None: + body["path_filter"] = self.path_filter + return body + + def as_shallow_dict(self) -> dict: + """Serializes the FileFilter into a shallow dictionary of its immediate attributes.""" + body = {} + if self.modified_after is not None: + body["modified_after"] = self.modified_after + if self.modified_before is not None: + body["modified_before"] = self.modified_before + if self.path_filter is not None: + body["path_filter"] = self.path_filter + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> FileFilter: + """Deserializes the FileFilter from a dictionary.""" + return cls( + modified_after=d.get("modified_after", None), + modified_before=d.get("modified_before", None), + path_filter=d.get("path_filter", None), + ) + + +@dataclass +class FileIngestionOptions: + corrupt_record_column: Optional[str] = None + + file_filters: Optional[List[FileFilter]] = None + """Generic options""" + + format: Optional[FileIngestionOptionsFileFormat] = None + """required for TableSpec""" + + format_options: Optional[Dict[str, str]] = None + """Format-specific options Based on + https://docs.databricks.com/aws/en/ingestion/cloud-object-storage/auto-loader/options#file-format-options""" + + ignore_corrupt_files: Optional[bool] = None + + infer_column_types: Optional[bool] = None + + reader_case_sensitive: Optional[bool] = None + """Column name case sensitivity + https://docs.databricks.com/aws/en/ingestion/cloud-object-storage/auto-loader/schema#change-case-sensitive-behavior""" + + rescued_data_column: Optional[str] = None + + schema_evolution_mode: Optional[FileIngestionOptionsSchemaEvolutionMode] = None + + schema_hints: Optional[str] = None + """Override inferred schema of specific columns Based on + https://docs.databricks.com/aws/en/ingestion/cloud-object-storage/auto-loader/schema#override-schema-inference-with-schema-hints""" + + single_variant_column: Optional[str] = None + + def as_dict(self) -> dict: + """Serializes the FileIngestionOptions into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.corrupt_record_column is not None: + body["corrupt_record_column"] = self.corrupt_record_column + if self.file_filters: + body["file_filters"] = [v.as_dict() for v in self.file_filters] + if self.format is not None: + body["format"] = self.format.value + if self.format_options: + body["format_options"] = self.format_options + if self.ignore_corrupt_files is not None: + body["ignore_corrupt_files"] = self.ignore_corrupt_files + if self.infer_column_types is not None: + body["infer_column_types"] = self.infer_column_types + if self.reader_case_sensitive is not None: + body["reader_case_sensitive"] = self.reader_case_sensitive + if self.rescued_data_column is not None: + body["rescued_data_column"] = self.rescued_data_column + if self.schema_evolution_mode is not None: + body["schema_evolution_mode"] = self.schema_evolution_mode.value + if self.schema_hints is not None: + body["schema_hints"] = self.schema_hints + if self.single_variant_column is not None: + body["single_variant_column"] = self.single_variant_column + return body + + def as_shallow_dict(self) -> dict: + """Serializes the FileIngestionOptions into a shallow dictionary of its immediate attributes.""" + body = {} + if self.corrupt_record_column is not None: + body["corrupt_record_column"] = self.corrupt_record_column + if self.file_filters: + body["file_filters"] = self.file_filters + if self.format is not None: + body["format"] = self.format + if self.format_options: + body["format_options"] = self.format_options + if self.ignore_corrupt_files is not None: + body["ignore_corrupt_files"] = self.ignore_corrupt_files + if self.infer_column_types is not None: + body["infer_column_types"] = self.infer_column_types + if self.reader_case_sensitive is not None: + body["reader_case_sensitive"] = self.reader_case_sensitive + if self.rescued_data_column is not None: + body["rescued_data_column"] = self.rescued_data_column + if self.schema_evolution_mode is not None: + body["schema_evolution_mode"] = self.schema_evolution_mode + if self.schema_hints is not None: + body["schema_hints"] = self.schema_hints + if self.single_variant_column is not None: + body["single_variant_column"] = self.single_variant_column + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> FileIngestionOptions: + """Deserializes the FileIngestionOptions from a dictionary.""" + return cls( + corrupt_record_column=d.get("corrupt_record_column", None), + file_filters=_repeated_dict(d, "file_filters", FileFilter), + format=_enum(d, "format", FileIngestionOptionsFileFormat), + format_options=d.get("format_options", None), + ignore_corrupt_files=d.get("ignore_corrupt_files", None), + infer_column_types=d.get("infer_column_types", None), + reader_case_sensitive=d.get("reader_case_sensitive", None), + rescued_data_column=d.get("rescued_data_column", None), + schema_evolution_mode=_enum(d, "schema_evolution_mode", FileIngestionOptionsSchemaEvolutionMode), + schema_hints=d.get("schema_hints", None), + single_variant_column=d.get("single_variant_column", None), + ) + + +class FileIngestionOptionsFileFormat(Enum): + + AVRO = "AVRO" + BINARYFILE = "BINARYFILE" + CSV = "CSV" + EXCEL = "EXCEL" + JSON = "JSON" + ORC = "ORC" + PARQUET = "PARQUET" + XML = "XML" + + +class FileIngestionOptionsSchemaEvolutionMode(Enum): + """Based on + https://docs.databricks.com/aws/en/ingestion/cloud-object-storage/auto-loader/schema#how-does-auto-loader-schema-evolution-work + """ + + ADD_NEW_COLUMNS = "ADD_NEW_COLUMNS" + ADD_NEW_COLUMNS_WITH_TYPE_WIDENING = "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING" + FAIL_ON_NEW_COLUMNS = "FAIL_ON_NEW_COLUMNS" + NONE = "NONE" + RESCUE = "RESCUE" + + @dataclass class FileLibrary: path: Optional[str] = None @@ -655,6 +898,105 @@ def from_dict(cls, d: Dict[str, Any]) -> GetUpdateResponse: return cls(update=_from_dict(d, "update", UpdateInfo)) +@dataclass +class GoogleAdsOptions: + """Google Ads specific options for ingestion (object-level). When set, these values override the + corresponding fields in GoogleAdsConfig (source_configurations).""" + + manager_account_id: str + """(Optional at this level) Manager Account ID (also called MCC Account ID) used to list and access + customer accounts under this manager account. Overrides GoogleAdsConfig.manager_account_id from + source_configurations when set.""" + + lookback_window_days: Optional[int] = None + """(Optional) Number of days to look back for report tables to capture late-arriving data. If not + specified, defaults to 30 days.""" + + sync_start_date: Optional[str] = None + """(Optional) Start date for the initial sync of report tables in YYYY-MM-DD format. This + determines the earliest date from which to sync historical data. If not specified, defaults to 2 + years of historical data.""" + + def as_dict(self) -> dict: + """Serializes the GoogleAdsOptions into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.lookback_window_days is not None: + body["lookback_window_days"] = self.lookback_window_days + if self.manager_account_id is not None: + body["manager_account_id"] = self.manager_account_id + if self.sync_start_date is not None: + body["sync_start_date"] = self.sync_start_date + return body + + def as_shallow_dict(self) -> dict: + """Serializes the GoogleAdsOptions into a shallow dictionary of its immediate attributes.""" + body = {} + if self.lookback_window_days is not None: + body["lookback_window_days"] = self.lookback_window_days + if self.manager_account_id is not None: + body["manager_account_id"] = self.manager_account_id + if self.sync_start_date is not None: + body["sync_start_date"] = self.sync_start_date + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> GoogleAdsOptions: + """Deserializes the GoogleAdsOptions from a dictionary.""" + return cls( + lookback_window_days=d.get("lookback_window_days", None), + manager_account_id=d.get("manager_account_id", None), + sync_start_date=d.get("sync_start_date", None), + ) + + +@dataclass +class GoogleDriveOptions: + entity_type: Optional[GoogleDriveOptionsGoogleDriveEntityType] = None + + file_ingestion_options: Optional[FileIngestionOptions] = None + + url: Optional[str] = None + """Google Drive URL.""" + + def as_dict(self) -> dict: + """Serializes the GoogleDriveOptions into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.entity_type is not None: + body["entity_type"] = self.entity_type.value + if self.file_ingestion_options: + body["file_ingestion_options"] = self.file_ingestion_options.as_dict() + if self.url is not None: + body["url"] = self.url + return body + + def as_shallow_dict(self) -> dict: + """Serializes the GoogleDriveOptions into a shallow dictionary of its immediate attributes.""" + body = {} + if self.entity_type is not None: + body["entity_type"] = self.entity_type + if self.file_ingestion_options: + body["file_ingestion_options"] = self.file_ingestion_options + if self.url is not None: + body["url"] = self.url + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> GoogleDriveOptions: + """Deserializes the GoogleDriveOptions from a dictionary.""" + return cls( + entity_type=_enum(d, "entity_type", GoogleDriveOptionsGoogleDriveEntityType), + file_ingestion_options=_from_dict(d, "file_ingestion_options", FileIngestionOptions), + url=d.get("url", None), + ) + + +class GoogleDriveOptionsGoogleDriveEntityType(Enum): + + FILE = "FILE" + FILE_METADATA = "FILE_METADATA" + PERMISSION = "PERMISSION" + + @dataclass class IngestionConfig: report: Optional[ReportSpec] = None @@ -1044,6 +1386,7 @@ class IngestionSourceType(Enum): DYNAMICS365 = "DYNAMICS365" FOREIGN_CATALOG = "FOREIGN_CATALOG" GA4_RAW_DATA = "GA4_RAW_DATA" + GOOGLE_DRIVE = "GOOGLE_DRIVE" MANAGED_POSTGRESQL = "MANAGED_POSTGRESQL" MYSQL = "MYSQL" NETSUITE = "NETSUITE" @@ -3032,6 +3375,9 @@ class SchemaSpec: are created in this destination schema. The pipeline fails If a table with the same name already exists.""" + connector_options: Optional[ConnectorOptions] = None + """(Optional) Source Specific Connector Options""" + source_catalog: Optional[str] = None """The source catalog name. Might be optional depending on the type of source.""" @@ -3043,6 +3389,8 @@ class SchemaSpec: def as_dict(self) -> dict: """Serializes the SchemaSpec into a dictionary suitable for use as a JSON request body.""" body = {} + if self.connector_options: + body["connector_options"] = self.connector_options.as_dict() if self.destination_catalog is not None: body["destination_catalog"] = self.destination_catalog if self.destination_schema is not None: @@ -3058,6 +3406,8 @@ def as_dict(self) -> dict: def as_shallow_dict(self) -> dict: """Serializes the SchemaSpec into a shallow dictionary of its immediate attributes.""" body = {} + if self.connector_options: + body["connector_options"] = self.connector_options if self.destination_catalog is not None: body["destination_catalog"] = self.destination_catalog if self.destination_schema is not None: @@ -3074,6 +3424,7 @@ def as_shallow_dict(self) -> dict: def from_dict(cls, d: Dict[str, Any]) -> SchemaSpec: """Deserializes the SchemaSpec from a dictionary.""" return cls( + connector_options=_from_dict(d, "connector_options", ConnectorOptions), destination_catalog=d.get("destination_catalog", None), destination_schema=d.get("destination_schema", None), source_catalog=d.get("source_catalog", None), @@ -3160,6 +3511,57 @@ def from_dict(cls, d: Dict[str, Any]) -> SerializedException: ) +@dataclass +class SharepointOptions: + entity_type: Optional[SharepointOptionsSharepointEntityType] = None + """(Optional) The type of SharePoint entity to ingest. If not specified, defaults to FILE.""" + + file_ingestion_options: Optional[FileIngestionOptions] = None + """(Optional) File ingestion options for processing files.""" + + url: Optional[str] = None + """Required. The SharePoint URL.""" + + def as_dict(self) -> dict: + """Serializes the SharepointOptions into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.entity_type is not None: + body["entity_type"] = self.entity_type.value + if self.file_ingestion_options: + body["file_ingestion_options"] = self.file_ingestion_options.as_dict() + if self.url is not None: + body["url"] = self.url + return body + + def as_shallow_dict(self) -> dict: + """Serializes the SharepointOptions into a shallow dictionary of its immediate attributes.""" + body = {} + if self.entity_type is not None: + body["entity_type"] = self.entity_type + if self.file_ingestion_options: + body["file_ingestion_options"] = self.file_ingestion_options + if self.url is not None: + body["url"] = self.url + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> SharepointOptions: + """Deserializes the SharepointOptions from a dictionary.""" + return cls( + entity_type=_enum(d, "entity_type", SharepointOptionsSharepointEntityType), + file_ingestion_options=_from_dict(d, "file_ingestion_options", FileIngestionOptions), + url=d.get("url", None), + ) + + +class SharepointOptionsSharepointEntityType(Enum): + + FILE = "FILE" + FILE_METADATA = "FILE_METADATA" + LIST = "LIST" + PERMISSION = "PERMISSION" + + @dataclass class SourceCatalogConfig: """SourceCatalogConfig contains catalog-level custom configuration parameters for each source""" @@ -3337,6 +3739,9 @@ class TableSpec: destination_schema: str """Required. Destination schema to store table.""" + connector_options: Optional[ConnectorOptions] = None + """(Optional) Source Specific Connector Options""" + destination_table: Optional[str] = None """Optional. Destination table name. The pipeline fails if a table with that name already exists. If not set, the source table name is used.""" @@ -3354,6 +3759,8 @@ class TableSpec: def as_dict(self) -> dict: """Serializes the TableSpec into a dictionary suitable for use as a JSON request body.""" body = {} + if self.connector_options: + body["connector_options"] = self.connector_options.as_dict() if self.destination_catalog is not None: body["destination_catalog"] = self.destination_catalog if self.destination_schema is not None: @@ -3373,6 +3780,8 @@ def as_dict(self) -> dict: def as_shallow_dict(self) -> dict: """Serializes the TableSpec into a shallow dictionary of its immediate attributes.""" body = {} + if self.connector_options: + body["connector_options"] = self.connector_options if self.destination_catalog is not None: body["destination_catalog"] = self.destination_catalog if self.destination_schema is not None: @@ -3393,6 +3802,7 @@ def as_shallow_dict(self) -> dict: def from_dict(cls, d: Dict[str, Any]) -> TableSpec: """Deserializes the TableSpec from a dictionary.""" return cls( + connector_options=_from_dict(d, "connector_options", ConnectorOptions), destination_catalog=d.get("destination_catalog", None), destination_schema=d.get("destination_schema", None), destination_table=d.get("destination_table", None), @@ -3440,7 +3850,6 @@ class TableSpecificConfig: valid for the Salesforce connector""" scd_type: Optional[TableSpecificConfigScdType] = None - """The SCD type to use to ingest the table.""" sequence_by: Optional[List[str]] = None """The column names specifying the logical order of events in the source data. Spark Declarative @@ -3530,6 +3939,110 @@ class TableSpecificConfigScdType(Enum): SCD_TYPE_2 = "SCD_TYPE_2" +@dataclass +class TikTokAdsOptions: + """TikTok Ads specific options for ingestion""" + + data_level: Optional[TikTokAdsOptionsTikTokDataLevel] = None + """(Optional) Data level for the report. If not specified, defaults to AUCTION_CAMPAIGN.""" + + dimensions: Optional[List[str]] = None + """(Optional) Dimensions to include in the report. Examples: "campaign_id", "adgroup_id", "ad_id", + "stat_time_day", "stat_time_hour" If not specified, defaults to campaign_id.""" + + lookback_window_days: Optional[int] = None + """(Optional) Number of days to look back for report tables during incremental sync to capture + late-arriving conversions and attribution data. If not specified, defaults to 7 days.""" + + metrics: Optional[List[str]] = None + """(Optional) Metrics to include in the report. Examples: "spend", "impressions", "clicks", + "conversion", "cpc" If not specified, defaults to basic metrics (spend, impressions, clicks, + etc.)""" + + query_lifetime: Optional[bool] = None + """(Optional) Whether to request lifetime metrics (all-time aggregated data). When true, the report + returns all-time data. If not specified, defaults to false.""" + + report_type: Optional[TikTokAdsOptionsTikTokReportType] = None + """(Optional) Report type for the TikTok Ads API. If not specified, defaults to BASIC.""" + + sync_start_date: Optional[str] = None + """(Optional) Start date for the initial sync of report tables in YYYY-MM-DD format. This + determines the earliest date from which to sync historical data. If not specified, defaults to 1 + year of historical data for daily reports and 30 days for hourly reports.""" + + def as_dict(self) -> dict: + """Serializes the TikTokAdsOptions into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.data_level is not None: + body["data_level"] = self.data_level.value + if self.dimensions: + body["dimensions"] = [v for v in self.dimensions] + if self.lookback_window_days is not None: + body["lookback_window_days"] = self.lookback_window_days + if self.metrics: + body["metrics"] = [v for v in self.metrics] + if self.query_lifetime is not None: + body["query_lifetime"] = self.query_lifetime + if self.report_type is not None: + body["report_type"] = self.report_type.value + if self.sync_start_date is not None: + body["sync_start_date"] = self.sync_start_date + return body + + def as_shallow_dict(self) -> dict: + """Serializes the TikTokAdsOptions into a shallow dictionary of its immediate attributes.""" + body = {} + if self.data_level is not None: + body["data_level"] = self.data_level + if self.dimensions: + body["dimensions"] = self.dimensions + if self.lookback_window_days is not None: + body["lookback_window_days"] = self.lookback_window_days + if self.metrics: + body["metrics"] = self.metrics + if self.query_lifetime is not None: + body["query_lifetime"] = self.query_lifetime + if self.report_type is not None: + body["report_type"] = self.report_type + if self.sync_start_date is not None: + body["sync_start_date"] = self.sync_start_date + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> TikTokAdsOptions: + """Deserializes the TikTokAdsOptions from a dictionary.""" + return cls( + data_level=_enum(d, "data_level", TikTokAdsOptionsTikTokDataLevel), + dimensions=d.get("dimensions", None), + lookback_window_days=d.get("lookback_window_days", None), + metrics=d.get("metrics", None), + query_lifetime=d.get("query_lifetime", None), + report_type=_enum(d, "report_type", TikTokAdsOptionsTikTokReportType), + sync_start_date=d.get("sync_start_date", None), + ) + + +class TikTokAdsOptionsTikTokDataLevel(Enum): + """Data level for TikTok Ads report aggregation.""" + + AUCTION_AD = "AUCTION_AD" + AUCTION_ADGROUP = "AUCTION_ADGROUP" + AUCTION_ADVERTISER = "AUCTION_ADVERTISER" + AUCTION_CAMPAIGN = "AUCTION_CAMPAIGN" + + +class TikTokAdsOptionsTikTokReportType(Enum): + """Report type for TikTok Ads API.""" + + AUDIENCE = "AUDIENCE" + BASIC = "BASIC" + BUSINESS_CENTER = "BUSINESS_CENTER" + DSA = "DSA" + GMV_MAX = "GMV_MAX" + PLAYABLE_AD = "PLAYABLE_AD" + + @dataclass class Truncation: """Information about truncations applied to this event.""" @@ -3837,6 +4350,26 @@ def wait_get_pipeline_idle( attempt += 1 raise TimeoutError(f"timed out after {timeout}: {status_message}") + def apply_environment(self, pipeline_id: str) -> ApplyEnvironmentRequestResponse: + """* Applies the current pipeline environment onto the pipeline compute. The environment applied can be + used by subsequent dev-mode updates. + + :param pipeline_id: str + + :returns: :class:`ApplyEnvironmentRequestResponse` + """ + + headers = { + "Accept": "application/json", + } + + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Org-Id"] = cfg.workspace_id + + res = self._api.do("POST", f"/api/2.0/pipelines/{pipeline_id}/environment/apply", headers=headers) + return ApplyEnvironmentRequestResponse.from_dict(res) + def clone( self, pipeline_id: str, diff --git a/databricks/sdk/service/postgres.py b/databricks/sdk/service/postgres.py index dab0c26ae..dd5117490 100755 --- a/databricks/sdk/service/postgres.py +++ b/databricks/sdk/service/postgres.py @@ -2272,8 +2272,8 @@ class RoleRoleSpec: identity_type: * For the managed identities, OAUTH is used. * For the regular postgres roles, authentication based on postgres passwords is used. - NOTE: this is ignored for the Databricks identity type GROUP, and NO_LOGIN is implicitly assumed - instead for the GROUP identity type.""" + NOTE: for the Databricks identity type GROUP, LAKEBASE_OAUTH_V1 is the default auth method + (group can login as well).""" identity_type: Optional[RoleIdentityType] = None """The type of role. When specifying a managed-identity, the chosen role_id must be a valid: @@ -2419,6 +2419,7 @@ class SyncedTable: """Synced Table data synchronization status.""" uid: Optional[str] = None + """The Unity Catalog table ID for this synced table.""" def as_dict(self) -> dict: """Serializes the SyncedTable into a dictionary suitable for use as a JSON request body.""" @@ -2643,11 +2644,6 @@ class SyncedTableSyncedTableSpec: primary_key_columns: Optional[List[str]] = None """Primary Key columns to be used for data insert/update in the destination.""" - project: Optional[str] = None - """The full resource name of the project associated with the table. - - Format: "projects/{project_id}".""" - scheduling_policy: Optional[SyncedTableSyncedTableSpecSyncedTableSchedulingPolicy] = None """Scheduling policy of the underlying pipeline.""" @@ -2677,8 +2673,6 @@ def as_dict(self) -> dict: body["postgres_database"] = self.postgres_database if self.primary_key_columns: body["primary_key_columns"] = [v for v in self.primary_key_columns] - if self.project is not None: - body["project"] = self.project if self.scheduling_policy is not None: body["scheduling_policy"] = self.scheduling_policy.value if self.source_table_full_name is not None: @@ -2702,8 +2696,6 @@ def as_shallow_dict(self) -> dict: body["postgres_database"] = self.postgres_database if self.primary_key_columns: body["primary_key_columns"] = self.primary_key_columns - if self.project is not None: - body["project"] = self.project if self.scheduling_policy is not None: body["scheduling_policy"] = self.scheduling_policy if self.source_table_full_name is not None: @@ -2722,7 +2714,6 @@ def from_dict(cls, d: Dict[str, Any]) -> SyncedTableSyncedTableSpec: new_pipeline_spec=_from_dict(d, "new_pipeline_spec", NewPipelineSpec), postgres_database=d.get("postgres_database", None), primary_key_columns=d.get("primary_key_columns", None), - project=d.get("project", None), scheduling_policy=_enum(d, "scheduling_policy", SyncedTableSyncedTableSpecSyncedTableSchedulingPolicy), source_table_full_name=d.get("source_table_full_name", None), timeseries_key=d.get("timeseries_key", None), diff --git a/databricks/sdk/service/settings.py b/databricks/sdk/service/settings.py index 9997bd5d6..dffcf4381 100755 --- a/databricks/sdk/service/settings.py +++ b/databricks/sdk/service/settings.py @@ -866,6 +866,8 @@ class CreatePrivateEndpointRule: error_message: Optional[str] = None + gcp_endpoint: Optional[GcpEndpoint] = None + group_id: Optional[str] = None """Not used by customer-managed private endpoint services. @@ -891,6 +893,8 @@ def as_dict(self) -> dict: body["endpoint_service"] = self.endpoint_service if self.error_message is not None: body["error_message"] = self.error_message + if self.gcp_endpoint: + body["gcp_endpoint"] = self.gcp_endpoint.as_dict() if self.group_id is not None: body["group_id"] = self.group_id if self.resource_id is not None: @@ -908,6 +912,8 @@ def as_shallow_dict(self) -> dict: body["endpoint_service"] = self.endpoint_service if self.error_message is not None: body["error_message"] = self.error_message + if self.gcp_endpoint: + body["gcp_endpoint"] = self.gcp_endpoint if self.group_id is not None: body["group_id"] = self.group_id if self.resource_id is not None: @@ -923,6 +929,7 @@ def from_dict(cls, d: Dict[str, Any]) -> CreatePrivateEndpointRule: domain_names=d.get("domain_names", None), endpoint_service=d.get("endpoint_service", None), error_message=d.get("error_message", None), + gcp_endpoint=_from_dict(d, "gcp_endpoint", GcpEndpoint), group_id=d.get("group_id", None), resource_id=d.get("resource_id", None), resource_names=d.get("resource_names", None), @@ -1076,31 +1083,6 @@ def from_dict(cls, d: Dict[str, Any]) -> CustomerFacingIngressNetworkPolicy: return cls(public_access=_from_dict(d, "public_access", CustomerFacingIngressNetworkPolicyPublicAccess)) -@dataclass -class CustomerFacingIngressNetworkPolicyAppsDestination: - all_destinations: Optional[bool] = None - """Must be set to true.""" - - def as_dict(self) -> dict: - """Serializes the CustomerFacingIngressNetworkPolicyAppsDestination into a dictionary suitable for use as a JSON request body.""" - body = {} - if self.all_destinations is not None: - body["all_destinations"] = self.all_destinations - return body - - def as_shallow_dict(self) -> dict: - """Serializes the CustomerFacingIngressNetworkPolicyAppsDestination into a shallow dictionary of its immediate attributes.""" - body = {} - if self.all_destinations is not None: - body["all_destinations"] = self.all_destinations - return body - - @classmethod - def from_dict(cls, d: Dict[str, Any]) -> CustomerFacingIngressNetworkPolicyAppsDestination: - """Deserializes the CustomerFacingIngressNetworkPolicyAppsDestination from a dictionary.""" - return cls(all_destinations=d.get("all_destinations", None)) - - @dataclass class CustomerFacingIngressNetworkPolicyAuthentication: identities: Optional[List[CustomerFacingIngressNetworkPolicyAuthenticationIdentity]] = None @@ -1208,31 +1190,6 @@ def from_dict(cls, d: Dict[str, Any]) -> CustomerFacingIngressNetworkPolicyIpRan return cls(ip_ranges=d.get("ip_ranges", None)) -@dataclass -class CustomerFacingIngressNetworkPolicyLakebaseDestination: - all_destinations: Optional[bool] = None - """Must be set to true.""" - - def as_dict(self) -> dict: - """Serializes the CustomerFacingIngressNetworkPolicyLakebaseDestination into a dictionary suitable for use as a JSON request body.""" - body = {} - if self.all_destinations is not None: - body["all_destinations"] = self.all_destinations - return body - - def as_shallow_dict(self) -> dict: - """Serializes the CustomerFacingIngressNetworkPolicyLakebaseDestination into a shallow dictionary of its immediate attributes.""" - body = {} - if self.all_destinations is not None: - body["all_destinations"] = self.all_destinations - return body - - @classmethod - def from_dict(cls, d: Dict[str, Any]) -> CustomerFacingIngressNetworkPolicyLakebaseDestination: - """Deserializes the CustomerFacingIngressNetworkPolicyLakebaseDestination from a dictionary.""" - return cls(all_destinations=d.get("all_destinations", None)) - - @dataclass class CustomerFacingIngressNetworkPolicyPublicAccess: restriction_mode: CustomerFacingIngressNetworkPolicyPublicAccessRestrictionMode @@ -1382,23 +1339,16 @@ class CustomerFacingIngressNetworkPolicyRequestDestination: """When true, match all destinations, no other destination fields can be set. When not set or false, at least one specific destination must be provided.""" - apps: Optional[CustomerFacingIngressNetworkPolicyAppsDestination] = None - - lakebase: Optional[CustomerFacingIngressNetworkPolicyLakebaseDestination] = None - workspace_api: Optional[CustomerFacingIngressNetworkPolicyWorkspaceApiDestination] = None workspace_ui: Optional[CustomerFacingIngressNetworkPolicyWorkspaceUiDestination] = None + """Workspace destinations""" def as_dict(self) -> dict: """Serializes the CustomerFacingIngressNetworkPolicyRequestDestination into a dictionary suitable for use as a JSON request body.""" body = {} if self.all_destinations is not None: body["all_destinations"] = self.all_destinations - if self.apps: - body["apps"] = self.apps.as_dict() - if self.lakebase: - body["lakebase"] = self.lakebase.as_dict() if self.workspace_api: body["workspace_api"] = self.workspace_api.as_dict() if self.workspace_ui: @@ -1410,10 +1360,6 @@ def as_shallow_dict(self) -> dict: body = {} if self.all_destinations is not None: body["all_destinations"] = self.all_destinations - if self.apps: - body["apps"] = self.apps - if self.lakebase: - body["lakebase"] = self.lakebase if self.workspace_api: body["workspace_api"] = self.workspace_api if self.workspace_ui: @@ -1425,8 +1371,6 @@ def from_dict(cls, d: Dict[str, Any]) -> CustomerFacingIngressNetworkPolicyReque """Deserializes the CustomerFacingIngressNetworkPolicyRequestDestination from a dictionary.""" return cls( all_destinations=d.get("all_destinations", None), - apps=_from_dict(d, "apps", CustomerFacingIngressNetworkPolicyAppsDestination), - lakebase=_from_dict(d, "lakebase", CustomerFacingIngressNetworkPolicyLakebaseDestination), workspace_api=_from_dict(d, "workspace_api", CustomerFacingIngressNetworkPolicyWorkspaceApiDestination), workspace_ui=_from_dict(d, "workspace_ui", CustomerFacingIngressNetworkPolicyWorkspaceUiDestination), ) @@ -3346,6 +3290,41 @@ def from_dict(cls, d: Dict[str, Any]) -> FetchIpAccessListResponse: return cls(ip_access_list=_from_dict(d, "ip_access_list", IpAccessListInfo)) +@dataclass +class GcpEndpoint: + psc_endpoint_uri: Optional[str] = None + """Output only. The URI of the created PSC endpoint.""" + + service_attachment: Optional[str] = None + """The full url of the target service attachment. Example: + projects/my-gcp-project/regions/us-east4/serviceAttachments/my-service-attachment""" + + def as_dict(self) -> dict: + """Serializes the GcpEndpoint into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.psc_endpoint_uri is not None: + body["psc_endpoint_uri"] = self.psc_endpoint_uri + if self.service_attachment is not None: + body["service_attachment"] = self.service_attachment + return body + + def as_shallow_dict(self) -> dict: + """Serializes the GcpEndpoint into a shallow dictionary of its immediate attributes.""" + body = {} + if self.psc_endpoint_uri is not None: + body["psc_endpoint_uri"] = self.psc_endpoint_uri + if self.service_attachment is not None: + body["service_attachment"] = self.service_attachment + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> GcpEndpoint: + """Deserializes the GcpEndpoint from a dictionary.""" + return cls( + psc_endpoint_uri=d.get("psc_endpoint_uri", None), service_attachment=d.get("service_attachment", None) + ) + + @dataclass class GenericWebhookConfig: password: Optional[str] = None @@ -4517,6 +4496,8 @@ class NccPrivateEndpointRule: error_message: Optional[str] = None + gcp_endpoint: Optional[GcpEndpoint] = None + group_id: Optional[str] = None """Not used by customer-managed private endpoint services. @@ -4569,6 +4550,8 @@ def as_dict(self) -> dict: body["endpoint_service"] = self.endpoint_service if self.error_message is not None: body["error_message"] = self.error_message + if self.gcp_endpoint: + body["gcp_endpoint"] = self.gcp_endpoint.as_dict() if self.group_id is not None: body["group_id"] = self.group_id if self.network_connectivity_config_id is not None: @@ -4608,6 +4591,8 @@ def as_shallow_dict(self) -> dict: body["endpoint_service"] = self.endpoint_service if self.error_message is not None: body["error_message"] = self.error_message + if self.gcp_endpoint: + body["gcp_endpoint"] = self.gcp_endpoint if self.group_id is not None: body["group_id"] = self.group_id if self.network_connectivity_config_id is not None: @@ -4638,6 +4623,7 @@ def from_dict(cls, d: Dict[str, Any]) -> NccPrivateEndpointRule: endpoint_name=d.get("endpoint_name", None), endpoint_service=d.get("endpoint_service", None), error_message=d.get("error_message", None), + gcp_endpoint=_from_dict(d, "gcp_endpoint", GcpEndpoint), group_id=d.get("group_id", None), network_connectivity_config_id=d.get("network_connectivity_config_id", None), resource_id=d.get("resource_id", None), @@ -5620,6 +5606,8 @@ class UpdatePrivateEndpointRule: error_message: Optional[str] = None + gcp_endpoint: Optional[GcpEndpoint] = None + resource_names: Optional[List[str]] = None """Only used by private endpoints towards AWS S3 service. @@ -5636,6 +5624,8 @@ def as_dict(self) -> dict: body["enabled"] = self.enabled if self.error_message is not None: body["error_message"] = self.error_message + if self.gcp_endpoint: + body["gcp_endpoint"] = self.gcp_endpoint.as_dict() if self.resource_names: body["resource_names"] = [v for v in self.resource_names] return body @@ -5649,6 +5639,8 @@ def as_shallow_dict(self) -> dict: body["enabled"] = self.enabled if self.error_message is not None: body["error_message"] = self.error_message + if self.gcp_endpoint: + body["gcp_endpoint"] = self.gcp_endpoint if self.resource_names: body["resource_names"] = self.resource_names return body @@ -5660,6 +5652,7 @@ def from_dict(cls, d: Dict[str, Any]) -> UpdatePrivateEndpointRule: domain_names=d.get("domain_names", None), enabled=d.get("enabled", None), error_message=d.get("error_message", None), + gcp_endpoint=_from_dict(d, "gcp_endpoint", GcpEndpoint), resource_names=d.get("resource_names", None), ) @@ -9138,7 +9131,12 @@ def __init__(self, api_client): self._api = api_client def create_obo_token( - self, application_id: str, *, comment: Optional[str] = None, lifetime_seconds: Optional[int] = None + self, + application_id: str, + *, + comment: Optional[str] = None, + lifetime_seconds: Optional[int] = None, + scopes: Optional[List[str]] = None, ) -> CreateOboTokenResponse: """Creates a token on behalf of a service principal. @@ -9148,6 +9146,7 @@ def create_obo_token( Comment that describes the purpose of the token. :param lifetime_seconds: int (optional) The number of seconds before the token expires. + :param scopes: List[str] (optional) :returns: :class:`CreateOboTokenResponse` """ @@ -9159,6 +9158,8 @@ def create_obo_token( body["comment"] = comment if lifetime_seconds is not None: body["lifetime_seconds"] = lifetime_seconds + if scopes is not None: + body["scopes"] = [v for v in scopes] headers = { "Accept": "application/json", "Content-Type": "application/json", @@ -9333,7 +9334,13 @@ class TokensAPI: def __init__(self, api_client): self._api = api_client - def create(self, *, comment: Optional[str] = None, lifetime_seconds: Optional[int] = None) -> CreateTokenResponse: + def create( + self, + *, + comment: Optional[str] = None, + lifetime_seconds: Optional[int] = None, + scopes: Optional[List[str]] = None, + ) -> CreateTokenResponse: """Creates and returns a token for a user. If this call is made through token authentication, it creates a token with the same client ID as the authenticated token. If the user's token quota is exceeded, this call returns an error **QUOTA_EXCEEDED**. @@ -9344,6 +9351,8 @@ def create(self, *, comment: Optional[str] = None, lifetime_seconds: Optional[in The lifetime of the token, in seconds. If the lifetime is not specified, this token remains valid for 2 years. + :param scopes: List[str] (optional) + Optional scopes of the token. :returns: :class:`CreateTokenResponse` """ @@ -9353,6 +9362,8 @@ def create(self, *, comment: Optional[str] = None, lifetime_seconds: Optional[in body["comment"] = comment if lifetime_seconds is not None: body["lifetime_seconds"] = lifetime_seconds + if scopes is not None: + body["scopes"] = [v for v in scopes] headers = { "Accept": "application/json", "Content-Type": "application/json", diff --git a/databricks/sdk/service/sql.py b/databricks/sdk/service/sql.py index 9b4ac2962..5cfa69646 100755 --- a/databricks/sdk/service/sql.py +++ b/databricks/sdk/service/sql.py @@ -6209,6 +6209,10 @@ class StatementStatus: error: Optional[ServiceError] = None + sql_state: Optional[str] = None + """SQLSTATE error code returned when the statement execution fails. Only populated when the + statement status is `FAILED`.""" + state: Optional[StatementState] = None """Statement execution state: - `PENDING`: waiting for warehouse - `RUNNING`: running - `SUCCEEDED`: execution was successful, result data available for fetch - `FAILED`: execution @@ -6221,6 +6225,8 @@ def as_dict(self) -> dict: body = {} if self.error: body["error"] = self.error.as_dict() + if self.sql_state is not None: + body["sql_state"] = self.sql_state if self.state is not None: body["state"] = self.state.value return body @@ -6230,6 +6236,8 @@ def as_shallow_dict(self) -> dict: body = {} if self.error: body["error"] = self.error + if self.sql_state is not None: + body["sql_state"] = self.sql_state if self.state is not None: body["state"] = self.state return body @@ -6237,7 +6245,11 @@ def as_shallow_dict(self) -> dict: @classmethod def from_dict(cls, d: Dict[str, Any]) -> StatementStatus: """Deserializes the StatementStatus from a dictionary.""" - return cls(error=_from_dict(d, "error", ServiceError), state=_enum(d, "state", StatementState)) + return cls( + error=_from_dict(d, "error", ServiceError), + sql_state=d.get("sql_state", None), + state=_enum(d, "state", StatementState), + ) class Status(Enum): diff --git a/databricks/sdk/service/vectorsearch.py b/databricks/sdk/service/vectorsearch.py index 7cfe7346a..d8e204d7d 100755 --- a/databricks/sdk/service/vectorsearch.py +++ b/databricks/sdk/service/vectorsearch.py @@ -460,6 +460,11 @@ def from_dict(cls, d: Dict[str, Any]) -> EmbeddingVectorColumn: @dataclass class EndpointInfo: + budget_policy_id: Optional[str] = None + """Discussed here: https://databricks.atlassian.net/wiki/x/OQDlCQE Additional documentation: + https://aip.dev.databricks.com/129 the user selected budget policy id for the endpoint + (client-side)""" + creation_timestamp: Optional[int] = None """Timestamp of endpoint creation""" @@ -499,6 +504,8 @@ class EndpointInfo: def as_dict(self) -> dict: """Serializes the EndpointInfo into a dictionary suitable for use as a JSON request body.""" body = {} + if self.budget_policy_id is not None: + body["budget_policy_id"] = self.budget_policy_id if self.creation_timestamp is not None: body["creation_timestamp"] = self.creation_timestamp if self.creator is not None: @@ -528,6 +535,8 @@ def as_dict(self) -> dict: def as_shallow_dict(self) -> dict: """Serializes the EndpointInfo into a shallow dictionary of its immediate attributes.""" body = {} + if self.budget_policy_id is not None: + body["budget_policy_id"] = self.budget_policy_id if self.creation_timestamp is not None: body["creation_timestamp"] = self.creation_timestamp if self.creator is not None: @@ -558,6 +567,7 @@ def as_shallow_dict(self) -> dict: def from_dict(cls, d: Dict[str, Any]) -> EndpointInfo: """Deserializes the EndpointInfo from a dictionary.""" return cls( + budget_policy_id=d.get("budget_policy_id", None), creation_timestamp=d.get("creation_timestamp", None), creator=d.get("creator", None), custom_tags=_repeated_dict(d, "custom_tags", CustomTag), @@ -654,6 +664,18 @@ class EndpointType(Enum): """Type of endpoint.""" STANDARD = "STANDARD" + STORAGE_OPTIMIZED = "STORAGE_OPTIMIZED" + + +class IndexSubtype(Enum): + """The subtype of the vector search index, determining the indexing and retrieval strategy. - + `VECTOR`: Not supported. Use `HYBRID` instead. - `FULL_TEXT`: An index that uses full-text + search without vector embeddings. - `HYBRID`: An index that uses vector embeddings for + similarity search and hybrid search.""" + + FULL_TEXT = "FULL_TEXT" + HYBRID = "HYBRID" + VECTOR = "VECTOR" @dataclass @@ -940,6 +962,9 @@ class MiniVectorIndex: endpoint_name: Optional[str] = None """Name of the endpoint associated with the index""" + index_subtype: Optional[IndexSubtype] = None + """The subtype of the index.""" + index_type: Optional[VectorIndexType] = None name: Optional[str] = None @@ -955,6 +980,8 @@ def as_dict(self) -> dict: body["creator"] = self.creator if self.endpoint_name is not None: body["endpoint_name"] = self.endpoint_name + if self.index_subtype is not None: + body["index_subtype"] = self.index_subtype.value if self.index_type is not None: body["index_type"] = self.index_type.value if self.name is not None: @@ -970,6 +997,8 @@ def as_shallow_dict(self) -> dict: body["creator"] = self.creator if self.endpoint_name is not None: body["endpoint_name"] = self.endpoint_name + if self.index_subtype is not None: + body["index_subtype"] = self.index_subtype if self.index_type is not None: body["index_type"] = self.index_type if self.name is not None: @@ -984,6 +1013,7 @@ def from_dict(cls, d: Dict[str, Any]) -> MiniVectorIndex: return cls( creator=d.get("creator", None), endpoint_name=d.get("endpoint_name", None), + index_subtype=_enum(d, "index_subtype", IndexSubtype), index_type=_enum(d, "index_type", VectorIndexType), name=d.get("name", None), primary_key=d.get("primary_key", None), @@ -992,12 +1022,16 @@ def from_dict(cls, d: Dict[str, Any]) -> MiniVectorIndex: @dataclass class PatchEndpointBudgetPolicyResponse: + budget_policy_id: Optional[str] = None + effective_budget_policy_id: Optional[str] = None """The budget policy applied to the vector search endpoint.""" def as_dict(self) -> dict: """Serializes the PatchEndpointBudgetPolicyResponse into a dictionary suitable for use as a JSON request body.""" body = {} + if self.budget_policy_id is not None: + body["budget_policy_id"] = self.budget_policy_id if self.effective_budget_policy_id is not None: body["effective_budget_policy_id"] = self.effective_budget_policy_id return body @@ -1005,6 +1039,8 @@ def as_dict(self) -> dict: def as_shallow_dict(self) -> dict: """Serializes the PatchEndpointBudgetPolicyResponse into a shallow dictionary of its immediate attributes.""" body = {} + if self.budget_policy_id is not None: + body["budget_policy_id"] = self.budget_policy_id if self.effective_budget_policy_id is not None: body["effective_budget_policy_id"] = self.effective_budget_policy_id return body @@ -1012,7 +1048,10 @@ def as_shallow_dict(self) -> dict: @classmethod def from_dict(cls, d: Dict[str, Any]) -> PatchEndpointBudgetPolicyResponse: """Deserializes the PatchEndpointBudgetPolicyResponse from a dictionary.""" - return cls(effective_budget_policy_id=d.get("effective_budget_policy_id", None)) + return cls( + budget_policy_id=d.get("budget_policy_id", None), + effective_budget_policy_id=d.get("effective_budget_policy_id", None), + ) class PipelineType(Enum): @@ -1486,6 +1525,9 @@ class VectorIndex: endpoint_name: Optional[str] = None """Name of the endpoint associated with the index""" + index_subtype: Optional[IndexSubtype] = None + """The subtype of the index.""" + index_type: Optional[VectorIndexType] = None name: Optional[str] = None @@ -1507,6 +1549,8 @@ def as_dict(self) -> dict: body["direct_access_index_spec"] = self.direct_access_index_spec.as_dict() if self.endpoint_name is not None: body["endpoint_name"] = self.endpoint_name + if self.index_subtype is not None: + body["index_subtype"] = self.index_subtype.value if self.index_type is not None: body["index_type"] = self.index_type.value if self.name is not None: @@ -1528,6 +1572,8 @@ def as_shallow_dict(self) -> dict: body["direct_access_index_spec"] = self.direct_access_index_spec if self.endpoint_name is not None: body["endpoint_name"] = self.endpoint_name + if self.index_subtype is not None: + body["index_subtype"] = self.index_subtype if self.index_type is not None: body["index_type"] = self.index_type if self.name is not None: @@ -1546,6 +1592,7 @@ def from_dict(cls, d: Dict[str, Any]) -> VectorIndex: delta_sync_index_spec=_from_dict(d, "delta_sync_index_spec", DeltaSyncVectorIndexSpecResponse), direct_access_index_spec=_from_dict(d, "direct_access_index_spec", DirectAccessVectorIndexSpec), endpoint_name=d.get("endpoint_name", None), + index_subtype=_enum(d, "index_subtype", IndexSubtype), index_type=_enum(d, "index_type", VectorIndexType), name=d.get("name", None), primary_key=d.get("primary_key", None), @@ -1662,6 +1709,7 @@ def create_endpoint( *, budget_policy_id: Optional[str] = None, min_qps: Optional[int] = None, + usage_policy_id: Optional[str] = None, ) -> Wait[EndpointInfo]: """Create a new endpoint. @@ -1674,6 +1722,8 @@ def create_endpoint( :param min_qps: int (optional) Min QPS for the endpoint. Mutually exclusive with num_replicas. The actual replica count is calculated at index creation/sync time based on this value. + :param usage_policy_id: str (optional) + The usage policy id to be applied once we've migrated to usage policies :returns: Long-running operation waiter for :class:`EndpointInfo`. @@ -1689,6 +1739,8 @@ def create_endpoint( body["min_qps"] = min_qps if name is not None: body["name"] = name + if usage_policy_id is not None: + body["usage_policy_id"] = usage_policy_id headers = { "Accept": "application/json", "Content-Type": "application/json", @@ -1712,10 +1764,15 @@ def create_endpoint_and_wait( *, budget_policy_id: Optional[str] = None, min_qps: Optional[int] = None, + usage_policy_id: Optional[str] = None, timeout=timedelta(minutes=20), ) -> EndpointInfo: return self.create_endpoint( - budget_policy_id=budget_policy_id, endpoint_type=endpoint_type, min_qps=min_qps, name=name + budget_policy_id=budget_policy_id, + endpoint_type=endpoint_type, + min_qps=min_qps, + name=name, + usage_policy_id=usage_policy_id, ).result(timeout=timeout) def delete_endpoint(self, endpoint_name: str): @@ -1945,6 +2002,7 @@ def create_index( *, delta_sync_index_spec: Optional[DeltaSyncVectorIndexSpecRequest] = None, direct_access_index_spec: Optional[DirectAccessVectorIndexSpec] = None, + index_subtype: Optional[IndexSubtype] = None, ) -> VectorIndex: """Create a new index. @@ -1959,6 +2017,8 @@ def create_index( Specification for Delta Sync Index. Required if `index_type` is `DELTA_SYNC`. :param direct_access_index_spec: :class:`DirectAccessVectorIndexSpec` (optional) Specification for Direct Vector Access Index. Required if `index_type` is `DIRECT_ACCESS`. + :param index_subtype: :class:`IndexSubtype` (optional) + The subtype of the index. Use `HYBRID` or `FULL_TEXT`. `VECTOR` is not supported. :returns: :class:`VectorIndex` """ @@ -1970,6 +2030,8 @@ def create_index( body["direct_access_index_spec"] = direct_access_index_spec.as_dict() if endpoint_name is not None: body["endpoint_name"] = endpoint_name + if index_subtype is not None: + body["index_subtype"] = index_subtype.value if index_type is not None: body["index_type"] = index_type.value if name is not None: diff --git a/databricks/sdk/service/workspace.py b/databricks/sdk/service/workspace.py index 74157ef0e..a2ce77be2 100755 --- a/databricks/sdk/service/workspace.py +++ b/databricks/sdk/service/workspace.py @@ -2716,10 +2716,10 @@ def __init__(self, api_client): self._api = api_client def delete(self, path: str, *, recursive: Optional[bool] = None): - """Deletes an object or a directory (and optionally recursively deletes all objects in the directory). * - If `path` does not exist, this call returns an error `RESOURCE_DOES_NOT_EXIST`. * If `path` is a - non-empty directory and `recursive` is set to `false`, this call returns an error - `DIRECTORY_NOT_EMPTY`. + """Deprecated: use WorkspaceHierarchyService.DeleteTreeNode instead. Deletes an object or a directory + (and optionally recursively deletes all objects in the directory). * If `path` does not exist, this + call returns an error `RESOURCE_DOES_NOT_EXIST`. * If `path` is a non-empty directory and `recursive` + is set to `false`, this call returns an error `DIRECTORY_NOT_EMPTY`. Object deletion cannot be undone and deleting a directory recursively is not atomic. @@ -2849,8 +2849,8 @@ def get_permissions(self, workspace_object_type: str, workspace_object_id: str) return WorkspaceObjectPermissions.from_dict(res) def get_status(self, path: str) -> ObjectInfo: - """Gets the status of an object or a directory. If `path` does not exist, this call returns an error - `RESOURCE_DOES_NOT_EXIST`. + """Deprecated: use WorkspaceHierarchyService.GetTreeNode instead. Gets the status of an object or a + directory. If `path` does not exist, this call returns an error `RESOURCE_DOES_NOT_EXIST`. :param path: str The absolute path of the notebook or directory. @@ -2938,8 +2938,9 @@ def import_( self._api.do("POST", "/api/2.0/workspace/import", body=body, headers=headers) def list(self, path: str, *, notebooks_modified_after: Optional[int] = None) -> Iterator[ObjectInfo]: - """Lists the contents of a directory, or the object if it is not a directory. If the input path does not - exist, this call returns an error `RESOURCE_DOES_NOT_EXIST`. + """Deprecated: use WorkspaceHierarchyService.ListTreeNodes instead. Lists the contents of a directory, or + the object if it is not a directory. If the input path does not exist, this call returns an error + `RESOURCE_DOES_NOT_EXIST`. :param path: str The absolute path of the notebook or directory. @@ -2967,9 +2968,9 @@ def list(self, path: str, *, notebooks_modified_after: Optional[int] = None) -> return parsed if parsed is not None else [] def mkdirs(self, path: str): - """Creates the specified directory (and necessary parent directories if they do not exist). If there is - an object (not a directory) at any prefix of the input path, this call returns an error - `RESOURCE_ALREADY_EXISTS`. + """Deprecated: use WorkspaceHierarchyService.CreateTreeNode instead. Creates the specified directory (and + necessary parent directories if they do not exist). If there is an object (not a directory) at any + prefix of the input path, this call returns an error `RESOURCE_ALREADY_EXISTS`. Note that if this operation fails it may have succeeded in creating some of the necessary parent directories. diff --git a/docs/dbdataclasses/apps.rst b/docs/dbdataclasses/apps.rst index d165647fc..d00fe91ba 100755 --- a/docs/dbdataclasses/apps.rst +++ b/docs/dbdataclasses/apps.rst @@ -206,6 +206,11 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. py:class:: AppResourceAppAppPermission + + .. py:attribute:: CAN_USE + :value: "CAN_USE" + .. autoclass:: AppResourceDatabase :members: :undoc-members: @@ -360,6 +365,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: VOLUME :value: "VOLUME" +.. autoclass:: AppThumbnail + :members: + :undoc-members: + .. autoclass:: AppUpdate :members: :undoc-members: diff --git a/docs/dbdataclasses/catalog.rst b/docs/dbdataclasses/catalog.rst index 08dd86867..712f5d74f 100755 --- a/docs/dbdataclasses/catalog.rst +++ b/docs/dbdataclasses/catalog.rst @@ -109,6 +109,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: AzureEncryptionSettings + :members: + :undoc-members: + .. autoclass:: AzureManagedIdentity :members: :undoc-members: @@ -279,7 +283,7 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:class:: ConnectionType - Next Id: 72 + Next Id: 75 .. py:attribute:: BIGQUERY :value: "BIGQUERY" @@ -420,7 +424,7 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:class:: CredentialType - Next Id: 17 + Next Id: 18 .. py:attribute:: ANY_STATIC_CREDENTIAL :value: "ANY_STATIC_CREDENTIAL" @@ -688,6 +692,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: EncryptionSettings + :members: + :undoc-members: + .. autoclass:: EntityTagAssignment :members: :undoc-members: @@ -1507,7 +1515,7 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:class:: SecurableKind - Latest kind: CONNECTION_GOOGLE_DRIVE_SERVICE_ACCOUNT = 301; Next id: 302 + Latest kind: CONNECTION_VEEVA_VAULT_OAUTH_M2M = 311; Next id: 312 .. py:attribute:: TABLE_DB_STORAGE :value: "TABLE_DB_STORAGE" @@ -1539,6 +1547,9 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: TABLE_DELTA_UNIFORM_ICEBERG_EXTERNAL :value: "TABLE_DELTA_UNIFORM_ICEBERG_EXTERNAL" + .. py:attribute:: TABLE_DELTA_UNIFORM_ICEBERG_EXTERNAL_DELTASHARING + :value: "TABLE_DELTA_UNIFORM_ICEBERG_EXTERNAL_DELTASHARING" + .. py:attribute:: TABLE_DELTA_UNIFORM_ICEBERG_FOREIGN_DELTASHARING :value: "TABLE_DELTA_UNIFORM_ICEBERG_FOREIGN_DELTASHARING" diff --git a/docs/dbdataclasses/dashboards.rst b/docs/dbdataclasses/dashboards.rst index b15f76d85..9246809a0 100755 --- a/docs/dbdataclasses/dashboards.rst +++ b/docs/dbdataclasses/dashboards.rst @@ -121,6 +121,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: GenieListConversationCommentsResponse + :members: + :undoc-members: + .. autoclass:: GenieListConversationMessagesResponse :members: :undoc-members: @@ -137,6 +141,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: GenieListMessageCommentsResponse + :members: + :undoc-members: + .. autoclass:: GenieListSpacesResponse :members: :undoc-members: @@ -145,6 +153,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: GenieMessageComment + :members: + :undoc-members: + .. autoclass:: GenieQueryAttachment :members: :undoc-members: @@ -540,6 +552,29 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: FOLLOW_UP_QUESTION :value: "FOLLOW_UP_QUESTION" +.. autoclass:: Thought + :members: + :undoc-members: + +.. py:class:: ThoughtType + + ThoughtType. The possible values are: * `THOUGHT_TYPE_UNSPECIFIED`: Default value that should not be used. * `THOUGHT_TYPE_DESCRIPTION`: A high-level description of how the question was interpreted. * `THOUGHT_TYPE_UNDERSTANDING`: How ambiguous parts of the question were resolved. * `THOUGHT_TYPE_DATA_SOURCING`: Which tables or datasets were identified as relevant. * `THOUGHT_TYPE_INSTRUCTIONS`: Which author-defined instructions were referenced. * `THOUGHT_TYPE_STEPS`: The logical steps taken to compute the answer. The category of a Thought. Additional values may be added in the future. + + .. py:attribute:: THOUGHT_TYPE_DATA_SOURCING + :value: "THOUGHT_TYPE_DATA_SOURCING" + + .. py:attribute:: THOUGHT_TYPE_DESCRIPTION + :value: "THOUGHT_TYPE_DESCRIPTION" + + .. py:attribute:: THOUGHT_TYPE_INSTRUCTIONS + :value: "THOUGHT_TYPE_INSTRUCTIONS" + + .. py:attribute:: THOUGHT_TYPE_STEPS + :value: "THOUGHT_TYPE_STEPS" + + .. py:attribute:: THOUGHT_TYPE_UNDERSTANDING + :value: "THOUGHT_TYPE_UNDERSTANDING" + .. autoclass:: TrashDashboardResponse :members: :undoc-members: diff --git a/docs/dbdataclasses/jobs.rst b/docs/dbdataclasses/jobs.rst index c30601b22..7442b03cc 100755 --- a/docs/dbdataclasses/jobs.rst +++ b/docs/dbdataclasses/jobs.rst @@ -1022,7 +1022,7 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:class:: TerminationCodeCode - The code indicates why the run was terminated. Additional codes might be introduced in future releases. * `SUCCESS`: The run was completed successfully. * `SUCCESS_WITH_FAILURES`: The run was completed successfully but some child runs failed. * `USER_CANCELED`: The run was successfully canceled during execution by a user. * `CANCELED`: The run was canceled during execution by the Databricks platform; for example, if the maximum run duration was exceeded. * `SKIPPED`: Run was never executed, for example, if the upstream task run failed, the dependency type condition was not met, or there were no material tasks to execute. * `INTERNAL_ERROR`: The run encountered an unexpected error. Refer to the state message for further details. * `DRIVER_ERROR`: The run encountered an error while communicating with the Spark Driver. * `CLUSTER_ERROR`: The run failed due to a cluster error. Refer to the state message for further details. * `REPOSITORY_CHECKOUT_FAILED`: Failed to complete the checkout due to an error when communicating with the third party service. * `INVALID_CLUSTER_REQUEST`: The run failed because it issued an invalid request to start the cluster. * `WORKSPACE_RUN_LIMIT_EXCEEDED`: The workspace has reached the quota for the maximum number of concurrent active runs. Consider scheduling the runs over a larger time frame. * `FEATURE_DISABLED`: The run failed because it tried to access a feature unavailable for the workspace. * `CLUSTER_REQUEST_LIMIT_EXCEEDED`: The number of cluster creation, start, and upsize requests have exceeded the allotted rate limit. Consider spreading the run execution over a larger time frame. * `STORAGE_ACCESS_ERROR`: The run failed due to an error when accessing the customer blob storage. Refer to the state message for further details. * `RUN_EXECUTION_ERROR`: The run was completed with task failures. For more details, refer to the state message or run output. * `UNAUTHORIZED_ERROR`: The run failed due to a permission issue while accessing a resource. Refer to the state message for further details. * `LIBRARY_INSTALLATION_ERROR`: The run failed while installing the user-requested library. Refer to the state message for further details. The causes might include, but are not limited to: The provided library is invalid, there are insufficient permissions to install the library, and so forth. * `MAX_CONCURRENT_RUNS_EXCEEDED`: The scheduled run exceeds the limit of maximum concurrent runs set for the job. * `MAX_SPARK_CONTEXTS_EXCEEDED`: The run is scheduled on a cluster that has already reached the maximum number of contexts it is configured to create. See: [Link]. * `RESOURCE_NOT_FOUND`: A resource necessary for run execution does not exist. Refer to the state message for further details. * `INVALID_RUN_CONFIGURATION`: The run failed due to an invalid configuration. Refer to the state message for further details. * `CLOUD_FAILURE`: The run failed due to a cloud provider issue. Refer to the state message for further details. * `MAX_JOB_QUEUE_SIZE_EXCEEDED`: The run was skipped due to reaching the job level queue size limit. * `DISABLED`: The run was never executed because it was disabled explicitly by the user. * `BREAKING_CHANGE`: Run failed because of an intentional breaking change in Spark, but it will be retried with a mitigation config. + The code indicates why the run was terminated. Additional codes might be introduced in future releases. * `SUCCESS`: The run was completed successfully. * `SUCCESS_WITH_FAILURES`: The run was completed successfully but some child runs failed. * `USER_CANCELED`: The run was successfully canceled during execution by a user. * `CANCELED`: The run was canceled during execution by the Databricks platform; for example, if the maximum run duration was exceeded. * `SKIPPED`: Run was never executed, for example, if the upstream task run failed, the dependency type condition was not met, or there were no material tasks to execute. * `INTERNAL_ERROR`: The run encountered an unexpected error. Refer to the state message for further details. * `DRIVER_ERROR`: The run encountered an error while communicating with the Spark Driver. * `CLUSTER_ERROR`: The run failed due to a cluster error. Refer to the state message for further details. * `REPOSITORY_CHECKOUT_FAILED`: Failed to complete the checkout due to an error when communicating with the third party service. * `INVALID_CLUSTER_REQUEST`: The run failed because it issued an invalid request to start the cluster. * `WORKSPACE_RUN_LIMIT_EXCEEDED`: The workspace has reached the quota for the maximum number of concurrent active runs. Consider scheduling the runs over a larger time frame. * `FEATURE_DISABLED`: The run failed because it tried to access a feature unavailable for the workspace. * `CLUSTER_REQUEST_LIMIT_EXCEEDED`: The number of cluster creation, start, and upsize requests have exceeded the allotted rate limit. Consider spreading the run execution over a larger time frame. * `STORAGE_ACCESS_ERROR`: The run failed due to an error when accessing the customer blob storage. Refer to the state message for further details. * `RUN_EXECUTION_ERROR`: The run was completed with task failures. For more details, refer to the state message or run output. * `UNAUTHORIZED_ERROR`: The run failed due to a permission issue while accessing a resource. Refer to the state message for further details. * `LIBRARY_INSTALLATION_ERROR`: The run failed while installing the user-requested library. Refer to the state message for further details. The causes might include, but are not limited to: The provided library is invalid, there are insufficient permissions to install the library, and so forth. * `MAX_CONCURRENT_RUNS_EXCEEDED`: The scheduled run exceeds the limit of maximum concurrent runs set for the job. * `MAX_SPARK_CONTEXTS_EXCEEDED`: The run is scheduled on a cluster that has already reached the maximum number of contexts it is configured to create. See: [Link]. * `RESOURCE_NOT_FOUND`: A resource necessary for run execution does not exist. Refer to the state message for further details. * `INVALID_RUN_CONFIGURATION`: The run failed due to an invalid configuration. Refer to the state message for further details. * `CLOUD_FAILURE`: The run failed due to a cloud provider issue. Refer to the state message for further details. * `MAX_JOB_QUEUE_SIZE_EXCEEDED`: The run was skipped due to reaching the job level queue size limit. * `DISABLED`: The run was never executed because it was disabled explicitly by the user. * `BREAKING_CHANGE`: Run failed because of an intentional breaking change in Spark, but it will be retried with a mitigation config. * `CLUSTER_TERMINATED_BY_USER`: The run failed because the externally managed cluster entered an unusable state, likely due to the user terminating or restarting it outside the jobs service. [Link]: https://kb.databricks.com/en_US/notebooks/too-many-execution-contexts-are-open-right-now .. py:attribute:: BUDGET_POLICY_LIMIT_EXCEEDED diff --git a/docs/dbdataclasses/ml.rst b/docs/dbdataclasses/ml.rst index 70fd72ba8..8c9e07565 100755 --- a/docs/dbdataclasses/ml.rst +++ b/docs/dbdataclasses/ml.rst @@ -319,6 +319,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: FieldDefinition + :members: + :undoc-members: + .. autoclass:: FileInfo :members: :undoc-members: @@ -331,6 +335,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: FlatSchema + :members: + :undoc-members: + .. autoclass:: ForecastingExperiment :members: :undoc-members: @@ -863,6 +871,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: RequestSource + :members: + :undoc-members: + .. autoclass:: RestoreExperimentResponse :members: :undoc-members: @@ -914,6 +926,43 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. py:class:: ScalarDataType + + Scalar data types for request-time field definitions. Only flat (non-nested) types are supported. + + .. py:attribute:: BINARY + :value: "BINARY" + + .. py:attribute:: BOOLEAN + :value: "BOOLEAN" + + .. py:attribute:: DATE + :value: "DATE" + + .. py:attribute:: DECIMAL + :value: "DECIMAL" + + .. py:attribute:: DOUBLE + :value: "DOUBLE" + + .. py:attribute:: FLOAT + :value: "FLOAT" + + .. py:attribute:: INTEGER + :value: "INTEGER" + + .. py:attribute:: LONG + :value: "LONG" + + .. py:attribute:: SHORT + :value: "SHORT" + + .. py:attribute:: STRING + :value: "STRING" + + .. py:attribute:: TIMESTAMP + :value: "TIMESTAMP" + .. autoclass:: SchemaConfig :members: :undoc-members: diff --git a/docs/dbdataclasses/pipelines.rst b/docs/dbdataclasses/pipelines.rst index 6b499821f..d295210bc 100755 --- a/docs/dbdataclasses/pipelines.rst +++ b/docs/dbdataclasses/pipelines.rst @@ -4,6 +4,10 @@ Delta Live Tables These dataclasses are used in the SDK to represent API requests and responses for services in the ``databricks.sdk.service.pipelines`` module. .. py:currentmodule:: databricks.sdk.service.pipelines +.. autoclass:: ApplyEnvironmentRequestResponse + :members: + :undoc-members: + .. autoclass:: AutoFullRefreshPolicy :members: :undoc-members: @@ -23,6 +27,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: ConnectorOptions + :members: + :undoc-members: + .. py:class:: ConnectorType For certain database sources LakeFlow Connect offers both query based and cdc ingestion, ConnectorType can bse used to convey the type of ingestion. If connection_name is provided for database sources, we default to Query Based ingestion @@ -113,6 +121,59 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: FileFilter + :members: + :undoc-members: + +.. autoclass:: FileIngestionOptions + :members: + :undoc-members: + +.. py:class:: FileIngestionOptionsFileFormat + + .. py:attribute:: AVRO + :value: "AVRO" + + .. py:attribute:: BINARYFILE + :value: "BINARYFILE" + + .. py:attribute:: CSV + :value: "CSV" + + .. py:attribute:: EXCEL + :value: "EXCEL" + + .. py:attribute:: JSON + :value: "JSON" + + .. py:attribute:: ORC + :value: "ORC" + + .. py:attribute:: PARQUET + :value: "PARQUET" + + .. py:attribute:: XML + :value: "XML" + +.. py:class:: FileIngestionOptionsSchemaEvolutionMode + + Based on https://docs.databricks.com/aws/en/ingestion/cloud-object-storage/auto-loader/schema#how-does-auto-loader-schema-evolution-work + + .. py:attribute:: ADD_NEW_COLUMNS + :value: "ADD_NEW_COLUMNS" + + .. py:attribute:: ADD_NEW_COLUMNS_WITH_TYPE_WIDENING + :value: "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING" + + .. py:attribute:: FAIL_ON_NEW_COLUMNS + :value: "FAIL_ON_NEW_COLUMNS" + + .. py:attribute:: NONE + :value: "NONE" + + .. py:attribute:: RESCUE + :value: "RESCUE" + .. autoclass:: FileLibrary :members: :undoc-members: @@ -143,6 +204,25 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: GoogleAdsOptions + :members: + :undoc-members: + +.. autoclass:: GoogleDriveOptions + :members: + :undoc-members: + +.. py:class:: GoogleDriveOptionsGoogleDriveEntityType + + .. py:attribute:: FILE + :value: "FILE" + + .. py:attribute:: FILE_METADATA + :value: "FILE_METADATA" + + .. py:attribute:: PERMISSION + :value: "PERMISSION" + .. autoclass:: IngestionConfig :members: :undoc-members: @@ -181,6 +261,9 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: GA4_RAW_DATA :value: "GA4_RAW_DATA" + .. py:attribute:: GOOGLE_DRIVE + :value: "GOOGLE_DRIVE" + .. py:attribute:: MANAGED_POSTGRESQL :value: "MANAGED_POSTGRESQL" @@ -440,6 +523,24 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: SharepointOptions + :members: + :undoc-members: + +.. py:class:: SharepointOptionsSharepointEntityType + + .. py:attribute:: FILE + :value: "FILE" + + .. py:attribute:: FILE_METADATA + :value: "FILE_METADATA" + + .. py:attribute:: LIST + :value: "LIST" + + .. py:attribute:: PERMISSION + :value: "PERMISSION" + .. autoclass:: SourceCatalogConfig :members: :undoc-members: @@ -506,6 +607,48 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: SCD_TYPE_2 :value: "SCD_TYPE_2" +.. autoclass:: TikTokAdsOptions + :members: + :undoc-members: + +.. py:class:: TikTokAdsOptionsTikTokDataLevel + + Data level for TikTok Ads report aggregation. + + .. py:attribute:: AUCTION_AD + :value: "AUCTION_AD" + + .. py:attribute:: AUCTION_ADGROUP + :value: "AUCTION_ADGROUP" + + .. py:attribute:: AUCTION_ADVERTISER + :value: "AUCTION_ADVERTISER" + + .. py:attribute:: AUCTION_CAMPAIGN + :value: "AUCTION_CAMPAIGN" + +.. py:class:: TikTokAdsOptionsTikTokReportType + + Report type for TikTok Ads API. + + .. py:attribute:: AUDIENCE + :value: "AUDIENCE" + + .. py:attribute:: BASIC + :value: "BASIC" + + .. py:attribute:: BUSINESS_CENTER + :value: "BUSINESS_CENTER" + + .. py:attribute:: DSA + :value: "DSA" + + .. py:attribute:: GMV_MAX + :value: "GMV_MAX" + + .. py:attribute:: PLAYABLE_AD + :value: "PLAYABLE_AD" + .. autoclass:: Truncation :members: :undoc-members: diff --git a/docs/dbdataclasses/settings.rst b/docs/dbdataclasses/settings.rst index dbd326243..fa86225be 100755 --- a/docs/dbdataclasses/settings.rst +++ b/docs/dbdataclasses/settings.rst @@ -206,10 +206,6 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: -.. autoclass:: CustomerFacingIngressNetworkPolicyAppsDestination - :members: - :undoc-members: - .. autoclass:: CustomerFacingIngressNetworkPolicyAuthentication :members: :undoc-members: @@ -241,10 +237,6 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: -.. autoclass:: CustomerFacingIngressNetworkPolicyLakebaseDestination - :members: - :undoc-members: - .. autoclass:: CustomerFacingIngressNetworkPolicyPublicAccess :members: :undoc-members: @@ -589,6 +581,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: GcpEndpoint + :members: + :undoc-members: + .. autoclass:: GenericWebhookConfig :members: :undoc-members: diff --git a/docs/dbdataclasses/vectorsearch.rst b/docs/dbdataclasses/vectorsearch.rst index 041a90f14..2837e0a0f 100755 --- a/docs/dbdataclasses/vectorsearch.rst +++ b/docs/dbdataclasses/vectorsearch.rst @@ -100,6 +100,22 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: STANDARD :value: "STANDARD" + .. py:attribute:: STORAGE_OPTIMIZED + :value: "STORAGE_OPTIMIZED" + +.. py:class:: IndexSubtype + + The subtype of the vector search index, determining the indexing and retrieval strategy. - `VECTOR`: Not supported. Use `HYBRID` instead. - `FULL_TEXT`: An index that uses full-text search without vector embeddings. - `HYBRID`: An index that uses vector embeddings for similarity search and hybrid search. + + .. py:attribute:: FULL_TEXT + :value: "FULL_TEXT" + + .. py:attribute:: HYBRID + :value: "HYBRID" + + .. py:attribute:: VECTOR + :value: "VECTOR" + .. autoclass:: ListEndpointResponse :members: :undoc-members: diff --git a/docs/workspace/apps/apps.rst b/docs/workspace/apps/apps.rst index 5ba41efd4..f09f49192 100644 --- a/docs/workspace/apps/apps.rst +++ b/docs/workspace/apps/apps.rst @@ -68,6 +68,16 @@ :returns: :class:`App` + .. py:method:: delete_app_thumbnail(name: str) + + Deletes the thumbnail for an app. + + :param name: str + The name of the app. + + + + .. py:method:: delete_space(name: str) -> DeleteSpaceOperation Deletes an app space. @@ -261,6 +271,18 @@ :returns: :class:`App` + .. py:method:: update_app_thumbnail(name: str [, app_thumbnail: Optional[AppThumbnail]]) -> AppThumbnail + + Updates the thumbnail for an app. + + :param name: str + The name of the app. + :param app_thumbnail: :class:`AppThumbnail` (optional) + The app thumbnail to set. + + :returns: :class:`AppThumbnail` + + .. py:method:: update_permissions(app_name: str [, access_control_list: Optional[List[AppAccessControlRequest]]]) -> AppPermissions Updates the permissions on an app. Apps can inherit permissions from their root object. diff --git a/docs/workspace/catalog/catalogs.rst b/docs/workspace/catalog/catalogs.rst index 258f994d3..e278d0bf7 100755 --- a/docs/workspace/catalog/catalogs.rst +++ b/docs/workspace/catalog/catalogs.rst @@ -11,7 +11,7 @@ the workspaces in a Databricks account. Users in different workspaces can share access to the same data, depending on privileges granted centrally in Unity Catalog. - .. py:method:: create(name: str [, comment: Optional[str], connection_name: Optional[str], options: Optional[Dict[str, str]], properties: Optional[Dict[str, str]], provider_name: Optional[str], share_name: Optional[str], storage_root: Optional[str]]) -> CatalogInfo + .. py:method:: create(name: str [, comment: Optional[str], connection_name: Optional[str], managed_encryption_settings: Optional[EncryptionSettings], options: Optional[Dict[str, str]], properties: Optional[Dict[str, str]], provider_name: Optional[str], share_name: Optional[str], storage_root: Optional[str]]) -> CatalogInfo Usage: @@ -24,10 +24,10 @@ w = WorkspaceClient() - created_catalog = w.catalogs.create(name=f"sdk-{time.time_ns()}") + new_catalog = w.catalogs.create(name=f"sdk-{time.time_ns()}") # cleanup - w.catalogs.delete(name=created_catalog.name, force=True) + w.catalogs.delete(name=new_catalog.name, force=True) Creates a new catalog instance in the parent metastore if the caller is a metastore admin or has the **CREATE_CATALOG** privilege. @@ -38,6 +38,8 @@ User-provided free-form text description. :param connection_name: str (optional) The name of the connection to an external data source. + :param managed_encryption_settings: :class:`EncryptionSettings` (optional) + Control CMK encryption for managed catalog data :param options: Dict[str,str] (optional) A map of key-value properties attached to the securable. :param properties: Dict[str,str] (optional) @@ -145,7 +147,7 @@ :returns: Iterator over :class:`CatalogInfo` - .. py:method:: update(name: str [, comment: Optional[str], enable_predictive_optimization: Optional[EnablePredictiveOptimization], isolation_mode: Optional[CatalogIsolationMode], new_name: Optional[str], options: Optional[Dict[str, str]], owner: Optional[str], properties: Optional[Dict[str, str]]]) -> CatalogInfo + .. py:method:: update(name: str [, comment: Optional[str], enable_predictive_optimization: Optional[EnablePredictiveOptimization], isolation_mode: Optional[CatalogIsolationMode], managed_encryption_settings: Optional[EncryptionSettings], new_name: Optional[str], options: Optional[Dict[str, str]], owner: Optional[str], properties: Optional[Dict[str, str]]]) -> CatalogInfo Usage: @@ -177,6 +179,8 @@ Whether predictive optimization should be enabled for this object and objects under it. :param isolation_mode: :class:`CatalogIsolationMode` (optional) Whether the current securable is accessible from all workspaces or a specific set of workspaces. + :param managed_encryption_settings: :class:`EncryptionSettings` (optional) + Control CMK encryption for managed catalog data :param new_name: str (optional) New name for the catalog. :param options: Dict[str,str] (optional) diff --git a/docs/workspace/catalog/entity_tag_assignments.rst b/docs/workspace/catalog/entity_tag_assignments.rst index f1111bffe..fd6aedcb0 100644 --- a/docs/workspace/catalog/entity_tag_assignments.rst +++ b/docs/workspace/catalog/entity_tag_assignments.rst @@ -41,8 +41,7 @@ [Manage tag policy permissions]: https://docs.databricks.com/aws/en/admin/tag-policies/manage-permissions :param entity_type: str - The type of the entity to which the tag is assigned. Allowed values are: catalogs, schemas, tables, - columns, volumes. + The type of the entity to which the tag is assigned. :param entity_name: str The fully qualified name of the entity to which the tag is assigned :param tag_key: str @@ -56,8 +55,7 @@ Gets a tag assignment for an Unity Catalog entity by tag key. :param entity_type: str - The type of the entity to which the tag is assigned. Allowed values are: catalogs, schemas, tables, - columns, volumes. + The type of the entity to which the tag is assigned. :param entity_name: str The fully qualified name of the entity to which the tag is assigned :param tag_key: str @@ -75,8 +73,7 @@ which is the only indication that the end of results has been reached. :param entity_type: str - The type of the entity to which the tag is assigned. Allowed values are: catalogs, schemas, tables, - columns, volumes. + The type of the entity to which the tag is assigned. :param entity_name: str The fully qualified name of the entity to which the tag is assigned :param max_results: int (optional) @@ -101,8 +98,7 @@ [Manage tag policy permissions]: https://docs.databricks.com/aws/en/admin/tag-policies/manage-permissions :param entity_type: str - The type of the entity to which the tag is assigned. Allowed values are: catalogs, schemas, tables, - columns, volumes. + The type of the entity to which the tag is assigned. :param entity_name: str The fully qualified name of the entity to which the tag is assigned :param tag_key: str diff --git a/docs/workspace/catalog/external_locations.rst b/docs/workspace/catalog/external_locations.rst index 0578df8b4..41716522a 100755 --- a/docs/workspace/catalog/external_locations.rst +++ b/docs/workspace/catalog/external_locations.rst @@ -115,20 +115,20 @@ credential = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) created = w.external_locations.create( name=f"sdk-{time.time_ns()}", credential_name=credential.name, - url="s3://%s/%s" % (os.environ["TEST_BUCKET"], f"sdk-{time.time_ns()}"), + url=f's3://{os.environ["TEST_BUCKET"]}/sdk-{time.time_ns()}', ) - _ = w.external_locations.get(name=created.name) + _ = w.external_locations.get(get=created.name) # cleanup - w.storage_credentials.delete(name=credential.name) - w.external_locations.delete(name=created.name) + w.storage_credentials.delete(delete=credential.name) + w.external_locations.delete(delete=created.name) Gets an external location from the metastore. The caller must be either a metastore admin, the owner of the external location, or a user that has some privilege on the external location. @@ -200,24 +200,24 @@ credential = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) created = w.external_locations.create( name=f"sdk-{time.time_ns()}", credential_name=credential.name, - url="s3://%s/%s" % (os.environ["TEST_BUCKET"], f"sdk-{time.time_ns()}"), + url=f's3://{os.environ["TEST_BUCKET"]}/sdk-{time.time_ns()}', ) _ = w.external_locations.update( name=created.name, credential_name=credential.name, - url="s3://%s/%s" % (os.environ["TEST_BUCKET"], f"sdk-{time.time_ns()}"), + url=f's3://{os.environ["TEST_BUCKET"]}/sdk-{time.time_ns()}', ) # cleanup - w.storage_credentials.delete(name=credential.name) - w.external_locations.delete(name=created.name) + w.storage_credentials.delete(delete=credential.name) + w.external_locations.delete(delete=created.name) Updates an external location in the metastore. The caller must be the owner of the external location, or be a metastore admin. In the second case, the admin can only update the name of the external diff --git a/docs/workspace/catalog/schemas.rst b/docs/workspace/catalog/schemas.rst index 719d5a156..fd1479c78 100644 --- a/docs/workspace/catalog/schemas.rst +++ b/docs/workspace/catalog/schemas.rst @@ -22,13 +22,13 @@ w = WorkspaceClient() - created_catalog = w.catalogs.create(name=f"sdk-{time.time_ns()}") + new_catalog = w.catalogs.create(name=f"sdk-{time.time_ns()}") - created_schema = w.schemas.create(name=f"sdk-{time.time_ns()}", catalog_name=created_catalog.name) + created = w.schemas.create(name=f"sdk-{time.time_ns()}", catalog_name=new_catalog.name) # cleanup - w.catalogs.delete(name=created_catalog.name, force=True) - w.schemas.delete(full_name=created_schema.full_name) + w.catalogs.delete(name=new_catalog.name, force=True) + w.schemas.delete(full_name=created.full_name) Creates a new schema for catalog in the Metastore. The caller must be a metastore admin, or have the **CREATE_SCHEMA** privilege in the parent catalog. diff --git a/docs/workspace/catalog/storage_credentials.rst b/docs/workspace/catalog/storage_credentials.rst index c174e87a3..92da2c568 100755 --- a/docs/workspace/catalog/storage_credentials.rst +++ b/docs/workspace/catalog/storage_credentials.rst @@ -30,14 +30,13 @@ w = WorkspaceClient() - storage_credential = w.storage_credentials.create( + created = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), - comment="created via SDK", ) # cleanup - w.storage_credentials.delete(name=storage_credential.name) + w.storage_credentials.delete(name=created.name) Creates a new storage credential. @@ -99,13 +98,13 @@ created = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) - by_name = w.storage_credentials.get(name=created.name) + by_name = w.storage_credentials.get(get=created.name) # cleanup - w.storage_credentials.delete(name=created.name) + w.storage_credentials.delete(delete=created.name) Gets a storage credential from the metastore. The caller must be a metastore admin, the owner of the storage credential, or have some permission on the storage credential. @@ -124,11 +123,10 @@ .. code-block:: from databricks.sdk import WorkspaceClient - from databricks.sdk.service import catalog w = WorkspaceClient() - all = w.storage_credentials.list(catalog.ListStorageCredentialsRequest()) + all = w.storage_credentials.list() Gets an array of storage credentials (as __StorageCredentialInfo__ objects). The array is limited to only those storage credentials the caller has permission to access. If the caller is a metastore diff --git a/docs/workspace/dashboards/genie.rst b/docs/workspace/dashboards/genie.rst index cdeb36af8..114093512 100755 --- a/docs/workspace/dashboards/genie.rst +++ b/docs/workspace/dashboards/genie.rst @@ -29,6 +29,22 @@ .. py:method:: create_message_and_wait(space_id: str, conversation_id: str, content: str, timeout: datetime.timedelta = 0:20:00) -> GenieMessage + .. py:method:: create_message_comment(space_id: str, conversation_id: str, message_id: str, content: str) -> GenieMessageComment + + Create a comment on a conversation message. + + :param space_id: str + The ID associated with the Genie space. + :param conversation_id: str + The ID associated with the conversation. + :param message_id: str + The ID associated with the message. + :param content: str + Comment text content. + + :returns: :class:`GenieMessageComment` + + .. py:method:: create_space(warehouse_id: str, serialized_space: str [, description: Optional[str], parent_path: Optional[str], title: Optional[str]]) -> GenieSpace Creates a Genie space from a serialized payload. @@ -328,6 +344,22 @@ :returns: :class:`GenieSpace` + .. py:method:: list_conversation_comments(space_id: str, conversation_id: str [, page_size: Optional[int], page_token: Optional[str]]) -> GenieListConversationCommentsResponse + + List all comments across all messages in a conversation. + + :param space_id: str + The ID associated with the Genie space. + :param conversation_id: str + The ID associated with the conversation. + :param page_size: int (optional) + Maximum number of comments to return per page. + :param page_token: str (optional) + Pagination token for getting the next page of results. + + :returns: :class:`GenieListConversationCommentsResponse` + + .. py:method:: list_conversation_messages(space_id: str, conversation_id: str [, page_size: Optional[int], page_token: Optional[str]]) -> GenieListConversationMessagesResponse List messages in a conversation @@ -361,6 +393,24 @@ :returns: :class:`GenieListConversationsResponse` + .. py:method:: list_message_comments(space_id: str, conversation_id: str, message_id: str [, page_size: Optional[int], page_token: Optional[str]]) -> GenieListMessageCommentsResponse + + List comments on a specific conversation message. + + :param space_id: str + The ID associated with the Genie space. + :param conversation_id: str + The ID associated with the conversation. + :param message_id: str + The ID associated with the message. + :param page_size: int (optional) + Maximum number of comments to return per page. + :param page_token: str (optional) + Pagination token for getting the next page of results. + + :returns: :class:`GenieListMessageCommentsResponse` + + .. py:method:: list_spaces( [, page_size: Optional[int], page_token: Optional[str]]) -> GenieListSpacesResponse Get list of Genie Spaces. @@ -373,7 +423,7 @@ :returns: :class:`GenieListSpacesResponse` - .. py:method:: send_message_feedback(space_id: str, conversation_id: str, message_id: str, rating: GenieFeedbackRating) + .. py:method:: send_message_feedback(space_id: str, conversation_id: str, message_id: str, rating: GenieFeedbackRating [, comment: Optional[str]]) Send feedback for a message. @@ -385,6 +435,8 @@ The ID associated with the message to provide feedback for. :param rating: :class:`GenieFeedbackRating` The rating (POSITIVE, NEGATIVE, or NONE). + :param comment: str (optional) + Optional text feedback that will be stored as a comment. diff --git a/docs/workspace/iam/permissions.rst b/docs/workspace/iam/permissions.rst index bf8411654..98142aeb7 100755 --- a/docs/workspace/iam/permissions.rst +++ b/docs/workspace/iam/permissions.rst @@ -44,7 +44,7 @@ obj = w.workspace.get_status(path=notebook_path) - levels = w.permissions.get_permission_levels(request_object_type="notebooks", request_object_id="%d" % (obj.object_id)) + _ = w.permissions.get(request_object_type="notebooks", request_object_id="%d" % (obj.object_id)) Gets the permissions of an object. Objects can inherit permissions from their parent objects or root object. diff --git a/docs/workspace/ml/model_registry.rst b/docs/workspace/ml/model_registry.rst index 46c3a4565..c528f4329 100755 --- a/docs/workspace/ml/model_registry.rst +++ b/docs/workspace/ml/model_registry.rst @@ -90,9 +90,7 @@ w = WorkspaceClient() - model = w.model_registry.create_model(name=f"sdk-{time.time_ns()}") - - created = w.model_registry.create_model_version(name=model.registered_model.name, source="dbfs:/tmp") + created = w.model_registry.create_model(name=f"sdk-{time.time_ns()}") Creates a new registered model with the name specified in the request body. Throws `RESOURCE_ALREADY_EXISTS` if a registered model with the given name exists. @@ -122,7 +120,7 @@ model = w.model_registry.create_model(name=f"sdk-{time.time_ns()}") - created = w.model_registry.create_model_version(name=model.registered_model.name, source="dbfs:/tmp") + mv = w.model_registry.create_model_version(name=model.registered_model.name, source="dbfs:/tmp") Creates a model version. @@ -736,13 +734,14 @@ w = WorkspaceClient() - created = w.model_registry.create_model(name=f"sdk-{time.time_ns()}") + model = w.model_registry.create_model(name=f"sdk-{time.time_ns()}") - model = w.model_registry.get_model(name=created.registered_model.name) + created = w.model_registry.create_model_version(name=model.registered_model.name, source="dbfs:/tmp") - w.model_registry.update_model( - name=model.registered_model_databricks.name, + w.model_registry.update_model_version( description=f"sdk-{time.time_ns()}", + name=created.model_version.name, + version=created.model_version.version, ) Updates a registered model. diff --git a/docs/workspace/pipelines/pipelines.rst b/docs/workspace/pipelines/pipelines.rst index 8653bdf90..b4666a1b6 100755 --- a/docs/workspace/pipelines/pipelines.rst +++ b/docs/workspace/pipelines/pipelines.rst @@ -16,6 +16,16 @@ step. You can also enforce data quality with Spark Declarative Pipelines expectations. Expectations allow you to define expected data quality and specify how to handle records that fail those expectations. + .. py:method:: apply_environment(pipeline_id: str) -> ApplyEnvironmentRequestResponse + + * Applies the current pipeline environment onto the pipeline compute. The environment applied can be + used by subsequent dev-mode updates. + + :param pipeline_id: str + + :returns: :class:`ApplyEnvironmentRequestResponse` + + .. py:method:: clone(pipeline_id: str [, allow_duplicate_names: Optional[bool], budget_policy_id: Optional[str], catalog: Optional[str], channel: Optional[str], clone_mode: Optional[CloneMode], clusters: Optional[List[PipelineCluster]], configuration: Optional[Dict[str, str]], continuous: Optional[bool], deployment: Optional[PipelineDeployment], development: Optional[bool], edition: Optional[str], environment: Optional[PipelinesEnvironment], event_log: Optional[EventLogSpec], expected_last_modified: Optional[int], filters: Optional[Filters], gateway_definition: Optional[IngestionGatewayPipelineDefinition], id: Optional[str], ingestion_definition: Optional[IngestionPipelineDefinition], libraries: Optional[List[PipelineLibrary]], name: Optional[str], notifications: Optional[List[Notifications]], photon: Optional[bool], restart_window: Optional[RestartWindow], root_path: Optional[str], schema: Optional[str], serverless: Optional[bool], storage: Optional[str], tags: Optional[Dict[str, str]], target: Optional[str], trigger: Optional[PipelineTrigger], usage_policy_id: Optional[str]]) -> ClonePipelineResponse Creates a new pipeline using Unity Catalog from a pipeline using Hive Metastore. This method returns diff --git a/docs/workspace/settings/token_management.rst b/docs/workspace/settings/token_management.rst index ee0e831b8..911852dd7 100644 --- a/docs/workspace/settings/token_management.rst +++ b/docs/workspace/settings/token_management.rst @@ -7,7 +7,7 @@ Enables administrators to get all tokens and delete tokens for other users. Admins can either get every token, get a specific token by ID, or get all tokens for a particular user. - .. py:method:: create_obo_token(application_id: str [, comment: Optional[str], lifetime_seconds: Optional[int]]) -> CreateOboTokenResponse + .. py:method:: create_obo_token(application_id: str [, comment: Optional[str], lifetime_seconds: Optional[int], scopes: Optional[List[str]]]) -> CreateOboTokenResponse Usage: @@ -42,6 +42,7 @@ Comment that describes the purpose of the token. :param lifetime_seconds: int (optional) The number of seconds before the token expires. + :param scopes: List[str] (optional) :returns: :class:`CreateOboTokenResponse` diff --git a/docs/workspace/settings/tokens.rst b/docs/workspace/settings/tokens.rst index 4904ba652..1e7abd3cd 100755 --- a/docs/workspace/settings/tokens.rst +++ b/docs/workspace/settings/tokens.rst @@ -7,7 +7,7 @@ The Token API allows you to create, list, and revoke tokens that can be used to authenticate and access Databricks REST APIs. - .. py:method:: create( [, comment: Optional[str], lifetime_seconds: Optional[int]]) -> CreateTokenResponse + .. py:method:: create( [, comment: Optional[str], lifetime_seconds: Optional[int], scopes: Optional[List[str]]]) -> CreateTokenResponse Usage: @@ -35,6 +35,8 @@ The lifetime of the token, in seconds. If the lifetime is not specified, this token remains valid for 2 years. + :param scopes: List[str] (optional) + Optional scopes of the token. :returns: :class:`CreateTokenResponse` diff --git a/docs/workspace/vectorsearch/vector_search_endpoints.rst b/docs/workspace/vectorsearch/vector_search_endpoints.rst index 6857dd0fa..fe226147f 100755 --- a/docs/workspace/vectorsearch/vector_search_endpoints.rst +++ b/docs/workspace/vectorsearch/vector_search_endpoints.rst @@ -6,7 +6,7 @@ **Endpoint**: Represents the compute resources to host vector search indexes. - .. py:method:: create_endpoint(name: str, endpoint_type: EndpointType [, budget_policy_id: Optional[str], min_qps: Optional[int]]) -> Wait[EndpointInfo] + .. py:method:: create_endpoint(name: str, endpoint_type: EndpointType [, budget_policy_id: Optional[str], min_qps: Optional[int], usage_policy_id: Optional[str]]) -> Wait[EndpointInfo] Create a new endpoint. @@ -19,13 +19,15 @@ :param min_qps: int (optional) Min QPS for the endpoint. Mutually exclusive with num_replicas. The actual replica count is calculated at index creation/sync time based on this value. + :param usage_policy_id: str (optional) + The usage policy id to be applied once we've migrated to usage policies :returns: Long-running operation waiter for :class:`EndpointInfo`. See :method:wait_get_endpoint_vector_search_endpoint_online for more details. - .. py:method:: create_endpoint_and_wait(name: str, endpoint_type: EndpointType [, budget_policy_id: Optional[str], min_qps: Optional[int], timeout: datetime.timedelta = 0:20:00]) -> EndpointInfo + .. py:method:: create_endpoint_and_wait(name: str, endpoint_type: EndpointType [, budget_policy_id: Optional[str], min_qps: Optional[int], usage_policy_id: Optional[str], timeout: datetime.timedelta = 0:20:00]) -> EndpointInfo .. py:method:: delete_endpoint(endpoint_name: str) diff --git a/docs/workspace/vectorsearch/vector_search_indexes.rst b/docs/workspace/vectorsearch/vector_search_indexes.rst index a634d9dff..2316c9e6a 100644 --- a/docs/workspace/vectorsearch/vector_search_indexes.rst +++ b/docs/workspace/vectorsearch/vector_search_indexes.rst @@ -12,7 +12,7 @@ Delta Table changes. - **Direct Vector Access Index**: An index that supports direct read and write of vectors and metadata through our REST and SDK APIs. With this model, the user manages index updates. - .. py:method:: create_index(name: str, endpoint_name: str, primary_key: str, index_type: VectorIndexType [, delta_sync_index_spec: Optional[DeltaSyncVectorIndexSpecRequest], direct_access_index_spec: Optional[DirectAccessVectorIndexSpec]]) -> VectorIndex + .. py:method:: create_index(name: str, endpoint_name: str, primary_key: str, index_type: VectorIndexType [, delta_sync_index_spec: Optional[DeltaSyncVectorIndexSpecRequest], direct_access_index_spec: Optional[DirectAccessVectorIndexSpec], index_subtype: Optional[IndexSubtype]]) -> VectorIndex Create a new index. @@ -27,6 +27,8 @@ Specification for Delta Sync Index. Required if `index_type` is `DELTA_SYNC`. :param direct_access_index_spec: :class:`DirectAccessVectorIndexSpec` (optional) Specification for Direct Vector Access Index. Required if `index_type` is `DIRECT_ACCESS`. + :param index_subtype: :class:`IndexSubtype` (optional) + The subtype of the index. Use `HYBRID` or `FULL_TEXT`. `VECTOR` is not supported. :returns: :class:`VectorIndex` diff --git a/docs/workspace/workspace/workspace.rst b/docs/workspace/workspace/workspace.rst index 23362f2fd..7e0841f51 100755 --- a/docs/workspace/workspace/workspace.rst +++ b/docs/workspace/workspace/workspace.rst @@ -13,10 +13,10 @@ .. py:method:: delete(path: str [, recursive: Optional[bool]]) - Deletes an object or a directory (and optionally recursively deletes all objects in the directory). * - If `path` does not exist, this call returns an error `RESOURCE_DOES_NOT_EXIST`. * If `path` is a - non-empty directory and `recursive` is set to `false`, this call returns an error - `DIRECTORY_NOT_EMPTY`. + Deprecated: use WorkspaceHierarchyService.DeleteTreeNode instead. Deletes an object or a directory + (and optionally recursively deletes all objects in the directory). * If `path` does not exist, this + call returns an error `RESOURCE_DOES_NOT_EXIST`. * If `path` is a non-empty directory and `recursive` + is set to `false`, this call returns an error `DIRECTORY_NOT_EMPTY`. Object deletion cannot be undone and deleting a directory recursively is not atomic. @@ -150,12 +150,12 @@ w = WorkspaceClient() - notebook = f"/Users/{w.current_user.me().user_name}/sdk-{time.time_ns()}" + notebook_path = f"/Users/{w.current_user.me().user_name}/sdk-{time.time_ns()}" - get_status_response = w.workspace.get_status(path=notebook) + obj = w.workspace.get_status(path=notebook_path) - Gets the status of an object or a directory. If `path` does not exist, this call returns an error - `RESOURCE_DOES_NOT_EXIST`. + Deprecated: use WorkspaceHierarchyService.GetTreeNode instead. Gets the status of an object or a + directory. If `path` does not exist, this call returns an error `RESOURCE_DOES_NOT_EXIST`. :param path: str The absolute path of the notebook or directory. @@ -181,18 +181,11 @@ notebook_path = f"/Users/{w.current_user.me().user_name}/sdk-{time.time_ns()}" w.workspace.import_( - path=notebook_path, - overwrite=true_, + content=base64.b64encode(("CREATE LIVE TABLE dlt_sample AS SELECT 1").encode()).decode(), format=workspace.ImportFormat.SOURCE, - language=workspace.Language.PYTHON, - content=base64.b64encode( - ( - """import time - time.sleep(10) - dbutils.notebook.exit('hello') - """ - ).encode() - ).decode(), + language=workspace.Language.SQL, + overwrite=true_, + path=notebook_path, ) Imports a workspace object (for example, a notebook or file) or the contents of an entire directory. @@ -236,14 +229,16 @@ .. code-block:: + import os + import time + from databricks.sdk import WorkspaceClient w = WorkspaceClient() - names = [] - for i in w.workspace.list(f"/Users/{w.current_user.me().user_name}", recursive=True): - names.append(i.path) - assert len(names) > 0 + notebook = f"/Users/{w.current_user.me().user_name}/sdk-{time.time_ns()}" + + objects = w.workspace.list(path=os.path.dirname(notebook)) List workspace objects @@ -255,9 +250,9 @@ .. py:method:: mkdirs(path: str) - Creates the specified directory (and necessary parent directories if they do not exist). If there is - an object (not a directory) at any prefix of the input path, this call returns an error - `RESOURCE_ALREADY_EXISTS`. + Deprecated: use WorkspaceHierarchyService.CreateTreeNode instead. Creates the specified directory (and + necessary parent directories if they do not exist). If there is an object (not a directory) at any + prefix of the input path, this call returns an error `RESOURCE_ALREADY_EXISTS`. Note that if this operation fails it may have succeeded in creating some of the necessary parent directories.