From be1ed6184425cf2f896e004751dbc85e1dc1eca1 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 23:15:45 +0000 Subject: [PATCH 1/2] fix: add migration to convert RequestBodyPlainText with JSON content to request_body_json The Connector Builder UI sometimes generates RequestBodyPlainText for raw JSON string bodies. After CDK v7.17.1 (PR #971), RequestBodyPlainText is correctly routed to request_body_data (form-encoded), but this broke connectors where the Builder had misclassified JSON content as plain text. This adds a new manifest migration that detects RequestBodyPlainText where the value is a JSON-like string and converts it back to string-valued request_body_json, which is handled correctly by InterpolatedNestedRequestInputProvider. Co-Authored-By: bot_apk --- .../migrations/__init__.py | 4 + ...dy_plain_text_json_to_request_body_json.py | 80 ++++++ .../migrations/registry.yaml | 7 + unit_tests/manifest_migrations/conftest.py | 231 ++++++++++++++++++ .../test_manifest_migration.py | 19 ++ ..._request_body_plain_text_json_migration.py | 231 ++++++++++++++++++ 6 files changed, 572 insertions(+) create mode 100644 airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_plain_text_json_to_request_body_json.py create mode 100644 unit_tests/manifest_migrations/test_request_body_plain_text_json_migration.py diff --git a/airbyte_cdk/manifest_migrations/migrations/__init__.py b/airbyte_cdk/manifest_migrations/migrations/__init__.py index 4937a4853..87f7b7523 100644 --- a/airbyte_cdk/manifest_migrations/migrations/__init__.py +++ b/airbyte_cdk/manifest_migrations/migrations/__init__.py @@ -8,6 +8,9 @@ from airbyte_cdk.manifest_migrations.migrations.http_requester_request_body_json_data_to_request_body import ( HttpRequesterRequestBodyJsonDataToRequestBody, ) +from airbyte_cdk.manifest_migrations.migrations.http_requester_request_body_plain_text_json_to_request_body_json import ( + HttpRequesterRequestBodyPlainTextJsonToRequestBodyJson, +) from airbyte_cdk.manifest_migrations.migrations.http_requester_url_base_to_url import ( HttpRequesterUrlBaseToUrl, ) @@ -15,5 +18,6 @@ __all__ = [ "HttpRequesterPathToUrl", "HttpRequesterRequestBodyJsonDataToRequestBody", + "HttpRequesterRequestBodyPlainTextJsonToRequestBodyJson", "HttpRequesterUrlBaseToUrl", ] diff --git a/airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_plain_text_json_to_request_body_json.py b/airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_plain_text_json_to_request_body_json.py new file mode 100644 index 000000000..f114d8544 --- /dev/null +++ b/airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_plain_text_json_to_request_body_json.py @@ -0,0 +1,80 @@ +# +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.manifest_migrations.manifest_migration import ( + TYPE_TAG, + ManifestMigration, + ManifestType, +) + + +class HttpRequesterRequestBodyPlainTextJsonToRequestBodyJson(ManifestMigration): + """Migrate `RequestBodyPlainText` with JSON-like content back to `request_body_json`. + + The Connector Builder UI sometimes generates `request_body: {type: RequestBodyPlainText, ...}` + for raw JSON string bodies (Jinja templates containing JSON). After CDK v7.17.1 (PR #971), + `RequestBodyPlainText` is correctly routed to `request_body_data` (form-encoded), but this + broke connectors where the Builder had misclassified JSON content as plain text. + + This migration detects `RequestBodyPlainText` where the value is a JSON-like string and + converts it to `request_body_json` (a string-valued deprecated key that is handled correctly + by `InterpolatedNestedRequestInputProvider`). The existing + `HttpRequesterRequestBodyJsonDataToRequestBody` migration intentionally skips string-valued + `request_body_json`, so there is no conflict between the two migrations. + """ + + component_type = "HttpRequester" + request_body_key = "request_body" + request_body_json_key = "request_body_json" + plain_text_type = "RequestBodyPlainText" + + def should_migrate(self, manifest: ManifestType) -> bool: + if manifest.get(TYPE_TAG) != self.component_type: + return False + request_body = manifest.get(self.request_body_key) + if not isinstance(request_body, dict): + return False + if request_body.get("type") != self.plain_text_type: + return False + value = request_body.get("value") + if not isinstance(value, str): + return False + return self._is_json_like(value) + + def migrate(self, manifest: ManifestType) -> None: + value = manifest[self.request_body_key]["value"] + manifest.pop(self.request_body_key) + manifest[self.request_body_json_key] = value + + def validate(self, manifest: ManifestType) -> bool: + has_string_json = self.request_body_json_key in manifest and isinstance( + manifest[self.request_body_json_key], str + ) + has_request_body_plain_text = ( + isinstance(manifest.get(self.request_body_key), dict) + and manifest[self.request_body_key].get("type") == self.plain_text_type + and self._is_json_like(manifest[self.request_body_key].get("value", "")) + ) + return has_string_json and not has_request_body_plain_text + + @staticmethod + def _is_json_like(value: str) -> bool: + """Check if a string value looks like JSON content. + + Returns `True` when the stripped value starts with `{` or `[`, excluding + Jinja expression openers (`{{`) and Jinja block openers (`{%`). + """ + stripped = value.strip() + if not stripped: + return False + if stripped.startswith("["): + return True + if ( + stripped.startswith("{") + and not stripped.startswith("{{") + and not stripped.startswith("{%") + ): + return True + return False diff --git a/airbyte_cdk/manifest_migrations/migrations/registry.yaml b/airbyte_cdk/manifest_migrations/migrations/registry.yaml index 393f499d7..c041abfb7 100644 --- a/airbyte_cdk/manifest_migrations/migrations/registry.yaml +++ b/airbyte_cdk/manifest_migrations/migrations/registry.yaml @@ -20,3 +20,10 @@ manifest_migrations: description: | This migration updates the `request_body_json_data` field in the `http_requester` spec to `request_body`. The `request_body_json_data` field is deprecated and will be removed in a future version. + - name: http_requester_request_body_plain_text_json_to_request_body_json + order: 4 + description: | + This migration converts `RequestBodyPlainText` with JSON-like string content back to + string-valued `request_body_json`. The Connector Builder UI sometimes misclassifies + JSON string bodies as plain text, which after CDK v7.17.1 routes them to form-encoded + `request_body_data` instead of JSON. diff --git a/unit_tests/manifest_migrations/conftest.py b/unit_tests/manifest_migrations/conftest.py index fe9ecf04d..a89c7b358 100644 --- a/unit_tests/manifest_migrations/conftest.py +++ b/unit_tests/manifest_migrations/conftest.py @@ -1206,6 +1206,237 @@ def expected_manifest_with_migrated_to_request_body() -> Dict[str, Any]: } +@pytest.fixture +def manifest_with_request_body_plain_text_json() -> Dict[str, Any]: + return { + "version": "0.0.0", + "type": "DeclarativeSource", + "check": { + "type": "CheckStream", + "stream_names": ["A"], + }, + "definitions": { + "streams": { + "A": { + "type": "DeclarativeStream", + "name": "A", + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url": "https://api.example.com/v1/search", + "http_method": "POST", + "request_body": { + "type": "RequestBodyPlainText", + "value": '{"sort": [{"field": "createdAt", "order": "ASC"}], "filter": [{"type": "equals", "field": "active", "value": "true"}]}', + }, + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": []}, + }, + }, + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": {"$ref": "#/schemas/A"}, + }, + }, + "B": { + "type": "DeclarativeStream", + "name": "B", + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url": "https://api.example.com/v1/query", + "http_method": "POST", + "request_body": { + "type": "RequestBodyPlainText", + "value": "plain text body that is not JSON", + }, + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": []}, + }, + }, + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": {"$ref": "#/schemas/B"}, + }, + }, + }, + }, + "streams": [ + {"$ref": "#/definitions/streams/A"}, + {"$ref": "#/definitions/streams/B"}, + ], + "schemas": { + "A": { + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": True, + "properties": {"field_a1": {"type": "string"}}, + }, + "B": { + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": True, + "properties": {"field_b1": {"type": "string"}}, + }, + }, + } + + +@pytest.fixture +def expected_manifest_with_plain_text_json_migrated() -> Dict[str, Any]: + return { + "version": "0.0.0", + "type": "DeclarativeSource", + "check": {"type": "CheckStream", "stream_names": ["A"]}, + "definitions": { + "streams": { + "A": { + "type": "DeclarativeStream", + "name": "A", + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url": "https://api.example.com/v1/search", + "http_method": "POST", + "request_body_json": '{"sort": [{"field": "createdAt", "order": "ASC"}], "filter": [{"type": "equals", "field": "active", "value": "true"}]}', + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": []}, + }, + }, + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": { + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": True, + "properties": {"field_a1": {"type": "string"}}, + }, + }, + }, + "B": { + "type": "DeclarativeStream", + "name": "B", + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url": "https://api.example.com/v1/query", + "http_method": "POST", + "request_body": { + "type": "RequestBodyPlainText", + "value": "plain text body that is not JSON", + }, + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": []}, + }, + }, + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": { + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": True, + "properties": {"field_b1": {"type": "string"}}, + }, + }, + }, + }, + }, + "streams": [ + { + "type": "DeclarativeStream", + "name": "A", + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url": "https://api.example.com/v1/search", + "http_method": "POST", + "request_body_json": '{"sort": [{"field": "createdAt", "order": "ASC"}], "filter": [{"type": "equals", "field": "active", "value": "true"}]}', + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": []}, + }, + }, + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": { + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": True, + "properties": {"field_a1": {"type": "string"}}, + }, + }, + }, + { + "type": "DeclarativeStream", + "name": "B", + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url": "https://api.example.com/v1/query", + "http_method": "POST", + "request_body": { + "type": "RequestBodyPlainText", + "value": "plain text body that is not JSON", + }, + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": []}, + }, + }, + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": { + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": True, + "properties": {"field_b1": {"type": "string"}}, + }, + }, + }, + ], + "schemas": { + "A": { + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": True, + "properties": {"field_a1": {"type": "string"}}, + }, + "B": { + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": True, + "properties": {"field_b1": {"type": "string"}}, + }, + }, + "metadata": { + "applied_migrations": [ + { + "from_version": "0.0.0", + "to_version": "*", + "migration": "HttpRequesterRequestBodyPlainTextJsonToRequestBodyJson", + "migrated_at": "2025-04-01T00:00:00+00:00", + }, + ] + }, + } + + class DummyMigration(ManifestMigration): def _process_manifest(self, manifest): self.is_migrated = False diff --git a/unit_tests/manifest_migrations/test_manifest_migration.py b/unit_tests/manifest_migrations/test_manifest_migration.py index c5e2cebcc..e714a7e51 100644 --- a/unit_tests/manifest_migrations/test_manifest_migration.py +++ b/unit_tests/manifest_migrations/test_manifest_migration.py @@ -12,6 +12,7 @@ from airbyte_cdk.manifest_migrations.migrations import ( HttpRequesterPathToUrl, HttpRequesterRequestBodyJsonDataToRequestBody, + HttpRequesterRequestBodyPlainTextJsonToRequestBodyJson, HttpRequesterUrlBaseToUrl, ) from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ( @@ -69,6 +70,24 @@ def test_manifest_resolve_migrate_request_body_json_and_data_to_request_body( assert migrated_manifest == expected_manifest_with_migrated_to_request_body +@freeze_time("2025-04-01") +@patch.dict( + migrations_registry.MANIFEST_MIGRATIONS, + {"*": [HttpRequesterRequestBodyPlainTextJsonToRequestBodyJson]}, + clear=True, +) +def test_manifest_migrate_request_body_plain_text_json_to_request_body_json( + manifest_with_request_body_plain_text_json, + expected_manifest_with_plain_text_json_migrated, +) -> None: + """Verify that RequestBodyPlainText with JSON content is migrated to string-valued request_body_json.""" + + resolved_manifest = resolver.preprocess_manifest(manifest_with_request_body_plain_text_json) + migrated_manifest = ManifestMigrationHandler(dict(resolved_manifest)).apply_migrations() + + assert migrated_manifest == expected_manifest_with_plain_text_json_migrated + + @freeze_time("2025-04-01") @patch.dict(migrations_registry.MANIFEST_MIGRATIONS, {"0.0.0": [DummyMigration]}, clear=True) def test_manifest_resolve_do_not_migrate( diff --git a/unit_tests/manifest_migrations/test_request_body_plain_text_json_migration.py b/unit_tests/manifest_migrations/test_request_body_plain_text_json_migration.py new file mode 100644 index 000000000..c53240399 --- /dev/null +++ b/unit_tests/manifest_migrations/test_request_body_plain_text_json_migration.py @@ -0,0 +1,231 @@ +# +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. +# + +import pytest + +from airbyte_cdk.manifest_migrations.migrations.http_requester_request_body_plain_text_json_to_request_body_json import ( + HttpRequesterRequestBodyPlainTextJsonToRequestBodyJson, +) + + +@pytest.mark.parametrize( + "manifest,expected", + [ + pytest.param( + { + "type": "HttpRequester", + "request_body": { + "type": "RequestBodyPlainText", + "value": '{"sort": [{"field": "createdAt"}], "filter": []}', + }, + }, + True, + id="json_object_string", + ), + pytest.param( + { + "type": "HttpRequester", + "request_body": { + "type": "RequestBodyPlainText", + "value": '[{"field": "createdAt"}]', + }, + }, + True, + id="json_array_string", + ), + pytest.param( + { + "type": "HttpRequester", + "request_body": { + "type": "RequestBodyPlainText", + "value": ' {"sort": [{"field": "{{ config[\'sort_field\'] }}"}]} ', + }, + }, + True, + id="json_with_jinja_and_whitespace", + ), + pytest.param( + { + "type": "HttpRequester", + "request_body": { + "type": "RequestBodyPlainText", + "value": "plain text body content", + }, + }, + False, + id="actual_plain_text", + ), + pytest.param( + { + "type": "HttpRequester", + "request_body": { + "type": "RequestBodyPlainText", + "value": "interpolate_me=5&option={{ config['option'] }}", + }, + }, + False, + id="url_encoded_form_string", + ), + pytest.param( + { + "type": "HttpRequester", + "request_body": { + "type": "RequestBodyPlainText", + "value": "{{ config['body'] }}", + }, + }, + False, + id="jinja_expression_not_json", + ), + pytest.param( + { + "type": "HttpRequester", + "request_body": { + "type": "RequestBodyPlainText", + "value": "{% if true %}body{% endif %}", + }, + }, + False, + id="jinja_block_tag_not_json", + ), + pytest.param( + { + "type": "HttpRequester", + "request_body": { + "type": "RequestBodyJsonObject", + "value": {"key": "value"}, + }, + }, + False, + id="json_object_type_not_plain_text", + ), + pytest.param( + { + "type": "HttpRequester", + "request_body": { + "type": "RequestBodyUrlEncodedForm", + "value": {"key": "value"}, + }, + }, + False, + id="url_encoded_type_not_plain_text", + ), + pytest.param( + { + "type": "HttpRequester", + }, + False, + id="no_request_body", + ), + pytest.param( + { + "type": "HttpRequester", + "request_body": { + "type": "RequestBodyPlainText", + "value": "", + }, + }, + False, + id="empty_plain_text_value", + ), + pytest.param( + { + "type": "SomeOtherComponent", + "request_body": { + "type": "RequestBodyPlainText", + "value": '{"key": "value"}', + }, + }, + False, + id="wrong_component_type", + ), + ], +) +def test_should_migrate(manifest, expected): + migration = HttpRequesterRequestBodyPlainTextJsonToRequestBodyJson() + assert migration.should_migrate(manifest) == expected + + +@pytest.mark.parametrize( + "manifest,expected_manifest", + [ + pytest.param( + { + "type": "HttpRequester", + "url": "https://api.example.com/search", + "request_body": { + "type": "RequestBodyPlainText", + "value": '{"sort": [{"field": "createdAt"}], "filter": []}', + }, + }, + { + "type": "HttpRequester", + "url": "https://api.example.com/search", + "request_body_json": '{"sort": [{"field": "createdAt"}], "filter": []}', + }, + id="json_object_migrated_to_request_body_json", + ), + pytest.param( + { + "type": "HttpRequester", + "url": "https://api.example.com/search", + "request_body": { + "type": "RequestBodyPlainText", + "value": '{"sort": [{"field": "{{ config[\'sort_field\'] }}"}]}', + }, + }, + { + "type": "HttpRequester", + "url": "https://api.example.com/search", + "request_body_json": '{"sort": [{"field": "{{ config[\'sort_field\'] }}"}]}', + }, + id="json_with_jinja_migrated_to_request_body_json", + ), + pytest.param( + { + "type": "HttpRequester", + "url": "https://api.example.com/search", + "request_body": { + "type": "RequestBodyPlainText", + "value": '[{"id": 1}, {"id": 2}]', + }, + }, + { + "type": "HttpRequester", + "url": "https://api.example.com/search", + "request_body_json": '[{"id": 1}, {"id": 2}]', + }, + id="json_array_migrated_to_request_body_json", + ), + ], +) +def test_migrate(manifest, expected_manifest): + migration = HttpRequesterRequestBodyPlainTextJsonToRequestBodyJson() + assert migration.should_migrate(manifest) is True + migration.migrate(manifest) + assert manifest == expected_manifest + assert migration.validate(manifest) is True + + +def test_validate_after_migration(): + """Validate returns True when migration was applied correctly.""" + manifest = { + "type": "HttpRequester", + "request_body_json": '{"key": "value"}', + } + migration = HttpRequesterRequestBodyPlainTextJsonToRequestBodyJson() + assert migration.validate(manifest) is True + + +def test_validate_before_migration(): + """Validate returns False when migration has not been applied yet.""" + manifest = { + "type": "HttpRequester", + "request_body": { + "type": "RequestBodyPlainText", + "value": '{"key": "value"}', + }, + } + migration = HttpRequesterRequestBodyPlainTextJsonToRequestBodyJson() + assert migration.validate(manifest) is False From 4c6d3b3b9cd45c216c799489d3794a076bd6977f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 5 May 2026 16:56:09 +0000 Subject: [PATCH 2/2] fix: extend RequestBodyJsonObject to accept string values and migrate RequestBodyPlainText with JSON content - Extend RequestBodyJsonObject schema to accept both dict and string values (anyOf: [object, string]) to support JSON string templates with Jinja - Update Pydantic model to Union[Dict[str, Any], str] - Refactor migration to convert RequestBodyPlainText+JSON -> RequestBodyJsonObject (string value) instead of deprecated request_body_json - No routing changes needed: InterpolatedNestedRequestInputProvider already handles both strings and dicts Resolves https://github.com/airbytehq/oncall/issues/12007 Co-Authored-By: bot_apk --- ...dy_plain_text_json_to_request_body_json.py | 35 ++++++++++--------- .../migrations/registry.yaml | 8 ++--- .../declarative_component_schema.yaml | 8 +++-- .../models/declarative_component_schema.py | 2 +- unit_tests/manifest_migrations/conftest.py | 10 ++++-- .../test_manifest_migration.py | 2 +- ..._request_body_plain_text_json_migration.py | 26 ++++++++++---- 7 files changed, 57 insertions(+), 34 deletions(-) diff --git a/airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_plain_text_json_to_request_body_json.py b/airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_plain_text_json_to_request_body_json.py index f114d8544..11dd1c5e1 100644 --- a/airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_plain_text_json_to_request_body_json.py +++ b/airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_plain_text_json_to_request_body_json.py @@ -11,7 +11,7 @@ class HttpRequesterRequestBodyPlainTextJsonToRequestBodyJson(ManifestMigration): - """Migrate `RequestBodyPlainText` with JSON-like content back to `request_body_json`. + """Migrate `RequestBodyPlainText` with JSON-like content to `RequestBodyJsonObject`. The Connector Builder UI sometimes generates `request_body: {type: RequestBodyPlainText, ...}` for raw JSON string bodies (Jinja templates containing JSON). After CDK v7.17.1 (PR #971), @@ -19,16 +19,15 @@ class HttpRequesterRequestBodyPlainTextJsonToRequestBodyJson(ManifestMigration): broke connectors where the Builder had misclassified JSON content as plain text. This migration detects `RequestBodyPlainText` where the value is a JSON-like string and - converts it to `request_body_json` (a string-valued deprecated key that is handled correctly - by `InterpolatedNestedRequestInputProvider`). The existing - `HttpRequesterRequestBodyJsonDataToRequestBody` migration intentionally skips string-valued - `request_body_json`, so there is no conflict between the two migrations. + converts it to `RequestBodyJsonObject` with a string value. `RequestBodyJsonObject` now + accepts both dict and string values, and routes through `InterpolatedNestedRequestInputProvider` + which correctly handles string templates containing JSON. """ component_type = "HttpRequester" request_body_key = "request_body" - request_body_json_key = "request_body_json" plain_text_type = "RequestBodyPlainText" + json_object_type = "RequestBodyJsonObject" def should_migrate(self, manifest: ManifestType) -> bool: if manifest.get(TYPE_TAG) != self.component_type: @@ -44,20 +43,24 @@ def should_migrate(self, manifest: ManifestType) -> bool: return self._is_json_like(value) def migrate(self, manifest: ManifestType) -> None: - value = manifest[self.request_body_key]["value"] - manifest.pop(self.request_body_key) - manifest[self.request_body_json_key] = value + request_body = manifest[self.request_body_key] + request_body["type"] = self.json_object_type def validate(self, manifest: ManifestType) -> bool: - has_string_json = self.request_body_json_key in manifest and isinstance( - manifest[self.request_body_json_key], str + request_body = manifest.get(self.request_body_key) + if not isinstance(request_body, dict): + return False + is_json_object_with_string = ( + request_body.get("type") == self.json_object_type + and isinstance(request_body.get("value"), str) + and self._is_json_like(request_body.get("value", "")) ) - has_request_body_plain_text = ( - isinstance(manifest.get(self.request_body_key), dict) - and manifest[self.request_body_key].get("type") == self.plain_text_type - and self._is_json_like(manifest[self.request_body_key].get("value", "")) + is_plain_text_json = ( + request_body.get("type") == self.plain_text_type + and isinstance(request_body.get("value"), str) + and self._is_json_like(request_body.get("value", "")) ) - return has_string_json and not has_request_body_plain_text + return is_json_object_with_string and not is_plain_text_json @staticmethod def _is_json_like(value: str) -> bool: diff --git a/airbyte_cdk/manifest_migrations/migrations/registry.yaml b/airbyte_cdk/manifest_migrations/migrations/registry.yaml index c041abfb7..524a9af08 100644 --- a/airbyte_cdk/manifest_migrations/migrations/registry.yaml +++ b/airbyte_cdk/manifest_migrations/migrations/registry.yaml @@ -23,7 +23,7 @@ manifest_migrations: - name: http_requester_request_body_plain_text_json_to_request_body_json order: 4 description: | - This migration converts `RequestBodyPlainText` with JSON-like string content back to - string-valued `request_body_json`. The Connector Builder UI sometimes misclassifies - JSON string bodies as plain text, which after CDK v7.17.1 routes them to form-encoded - `request_body_data` instead of JSON. + This migration converts `RequestBodyPlainText` with JSON-like string content to + `RequestBodyJsonObject` with a string value. The Connector Builder UI sometimes + misclassifies JSON string bodies as plain text, which after CDK v7.17.1 routes them + to form-encoded `request_body_data` instead of JSON. diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 82e345ed2..e36df2863 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -4699,7 +4699,7 @@ definitions: type: string RequestBodyJsonObject: title: Json Object Body - description: Request body value converted into a JSON object + description: Request body value converted into a JSON object. Can be a dict or a Jinja-interpolated string that evaluates to a JSON object at runtime. type: object required: - type @@ -4709,8 +4709,10 @@ definitions: type: string enum: [RequestBodyJsonObject] value: - type: object - additionalProperties: true + anyOf: + - type: object + additionalProperties: true + - type: string RequestBodyGraphQL: title: GraphQL Body description: Request body value converted into a GraphQL query object diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 8bd7b1146..da023c8af 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1401,7 +1401,7 @@ class RequestBodyUrlEncodedForm(BaseModel): class RequestBodyJsonObject(BaseModel): type: Literal["RequestBodyJsonObject"] - value: Dict[str, Any] + value: Union[Dict[str, Any], str] class RequestBodyGraphQlQuery(BaseModel): diff --git a/unit_tests/manifest_migrations/conftest.py b/unit_tests/manifest_migrations/conftest.py index a89c7b358..647a3911c 100644 --- a/unit_tests/manifest_migrations/conftest.py +++ b/unit_tests/manifest_migrations/conftest.py @@ -1305,7 +1305,10 @@ def expected_manifest_with_plain_text_json_migrated() -> Dict[str, Any]: "type": "HttpRequester", "url": "https://api.example.com/v1/search", "http_method": "POST", - "request_body_json": '{"sort": [{"field": "createdAt", "order": "ASC"}], "filter": [{"type": "equals", "field": "active", "value": "true"}]}', + "request_body": { + "type": "RequestBodyJsonObject", + "value": '{"sort": [{"field": "createdAt", "order": "ASC"}], "filter": [{"type": "equals", "field": "active", "value": "true"}]}', + }, }, "record_selector": { "type": "RecordSelector", @@ -1363,7 +1366,10 @@ def expected_manifest_with_plain_text_json_migrated() -> Dict[str, Any]: "type": "HttpRequester", "url": "https://api.example.com/v1/search", "http_method": "POST", - "request_body_json": '{"sort": [{"field": "createdAt", "order": "ASC"}], "filter": [{"type": "equals", "field": "active", "value": "true"}]}', + "request_body": { + "type": "RequestBodyJsonObject", + "value": '{"sort": [{"field": "createdAt", "order": "ASC"}], "filter": [{"type": "equals", "field": "active", "value": "true"}]}', + }, }, "record_selector": { "type": "RecordSelector", diff --git a/unit_tests/manifest_migrations/test_manifest_migration.py b/unit_tests/manifest_migrations/test_manifest_migration.py index e714a7e51..40dce429b 100644 --- a/unit_tests/manifest_migrations/test_manifest_migration.py +++ b/unit_tests/manifest_migrations/test_manifest_migration.py @@ -80,7 +80,7 @@ def test_manifest_migrate_request_body_plain_text_json_to_request_body_json( manifest_with_request_body_plain_text_json, expected_manifest_with_plain_text_json_migrated, ) -> None: - """Verify that RequestBodyPlainText with JSON content is migrated to string-valued request_body_json.""" + """Verify that RequestBodyPlainText with JSON content is migrated to RequestBodyJsonObject with string value.""" resolved_manifest = resolver.preprocess_manifest(manifest_with_request_body_plain_text_json) migrated_manifest = ManifestMigrationHandler(dict(resolved_manifest)).apply_migrations() diff --git a/unit_tests/manifest_migrations/test_request_body_plain_text_json_migration.py b/unit_tests/manifest_migrations/test_request_body_plain_text_json_migration.py index c53240399..f4dd04acd 100644 --- a/unit_tests/manifest_migrations/test_request_body_plain_text_json_migration.py +++ b/unit_tests/manifest_migrations/test_request_body_plain_text_json_migration.py @@ -162,9 +162,12 @@ def test_should_migrate(manifest, expected): { "type": "HttpRequester", "url": "https://api.example.com/search", - "request_body_json": '{"sort": [{"field": "createdAt"}], "filter": []}', + "request_body": { + "type": "RequestBodyJsonObject", + "value": '{"sort": [{"field": "createdAt"}], "filter": []}', + }, }, - id="json_object_migrated_to_request_body_json", + id="json_object_migrated_to_request_body_json_object", ), pytest.param( { @@ -178,9 +181,12 @@ def test_should_migrate(manifest, expected): { "type": "HttpRequester", "url": "https://api.example.com/search", - "request_body_json": '{"sort": [{"field": "{{ config[\'sort_field\'] }}"}]}', + "request_body": { + "type": "RequestBodyJsonObject", + "value": '{"sort": [{"field": "{{ config[\'sort_field\'] }}"}]}', + }, }, - id="json_with_jinja_migrated_to_request_body_json", + id="json_with_jinja_migrated_to_request_body_json_object", ), pytest.param( { @@ -194,9 +200,12 @@ def test_should_migrate(manifest, expected): { "type": "HttpRequester", "url": "https://api.example.com/search", - "request_body_json": '[{"id": 1}, {"id": 2}]', + "request_body": { + "type": "RequestBodyJsonObject", + "value": '[{"id": 1}, {"id": 2}]', + }, }, - id="json_array_migrated_to_request_body_json", + id="json_array_migrated_to_request_body_json_object", ), ], ) @@ -212,7 +221,10 @@ def test_validate_after_migration(): """Validate returns True when migration was applied correctly.""" manifest = { "type": "HttpRequester", - "request_body_json": '{"key": "value"}', + "request_body": { + "type": "RequestBodyJsonObject", + "value": '{"key": "value"}', + }, } migration = HttpRequesterRequestBodyPlainTextJsonToRequestBodyJson() assert migration.validate(manifest) is True