From d06dca2c3cb5edb99c6be210978b3a0a97dddb20 Mon Sep 17 00:00:00 2001 From: Eyal Rozen Date: Thu, 11 Jun 2026 08:28:58 +0300 Subject: [PATCH 01/14] feat: add common kafka schema --- schemas/common/kafka/v1.schema.json | 108 ++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 schemas/common/kafka/v1.schema.json diff --git a/schemas/common/kafka/v1.schema.json b/schemas/common/kafka/v1.schema.json new file mode 100644 index 00000000..2bc574bb --- /dev/null +++ b/schemas/common/kafka/v1.schema.json @@ -0,0 +1,108 @@ +{ + "$id": "https://mapcolonies.com/common/kafka/v1", + "description": "Kafka configuration", + "type": "object", + "required": ["brokers", "enableSslAuth", "sslPaths", "kafkaConsumer", "kafkaProducer"], + "unevaluatedProperties": true, + "properties": { + "brokers": { + "type": "string", + "description": "Comma-separated list of broker addresses", + "x-env-value": "KAFKA_BROKERS" + }, + "clientId": { + "type": "string", + "description": "Client identifier", + "x-env-value": "KAFKA_CLIENT_ID" + }, + "enableSslAuth": { + "type": "boolean", + "description": "Enable SSL/TLS authentication", + "default": false, + "x-env-value": "KAFKA_ENABLE_SSL_AUTH" + }, + "sslPaths": { + "$ref": "#/definitions/sslPaths" + }, + "sasl": { + "$ref": "#/definitions/sasl" + }, + "kafkaConsumer": { + "type": "object", + "properties": { + "groupId": { + "type": "string", + "description": "Consumer group identifier", + "x-env-value": "KAFKA_GROUP_ID" + } + } + }, + "kafkaProducer": { + "type": "object" + }, + "outputTopic": { + "type": "string", + "description": "Kafka topic to produce messages to", + "x-env-value": "KAFKA_OUTPUT_TOPIC" + } + }, + "if": { + "properties": { + "enableSslAuth": { + "const": true + } + } + }, + "then": { + "properties": { + "sslPaths": { + "required": ["ca", "key", "cert"] + } + } + }, + "definitions": { + "sslPaths": { + "type": "object", + "description": "SSL certificate paths", + "unevaluatedProperties": false, + "properties": { + "ca": { + "type": "string", + "description": "Path to the CA certificate file", + "x-env-value": "KAFKA_CA_PATH" + }, + "key": { + "type": "string", + "description": "Path to the client key file", + "x-env-value": "KAFKA_KEY_PATH" + }, + "cert": { + "type": "string", + "description": "Path to the client certificate file", + "x-env-value": "KAFKA_CERT_PATH" + } + } + }, + "sasl": { + "type": "object", + "description": "SASL authentication configuration", + "properties": { + "mechanism": { + "type": "string", + "description": "SASL mechanism", + "x-env-value": "KAFKA_SASL_MECHANISM" + }, + "username": { + "type": "string", + "description": "SASL username", + "x-env-value": "KAFKA_SASL_USERNAME" + }, + "password": { + "type": "string", + "description": "SASL password", + "x-env-value": "KAFKA_SASL_PASSWORD" + } + } + } + } +} From cad4c11e126f263767ffb11d31390f98b7886469 Mon Sep 17 00:00:00 2001 From: Eyal Rozen Date: Thu, 11 Jun 2026 08:30:55 +0300 Subject: [PATCH 02/14] feat: add feedback-api schema --- schemas/vector/feedbackApi/v1.schema.json | 90 +++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 schemas/vector/feedbackApi/v1.schema.json diff --git a/schemas/vector/feedbackApi/v1.schema.json b/schemas/vector/feedbackApi/v1.schema.json new file mode 100644 index 00000000..3c28be3a --- /dev/null +++ b/schemas/vector/feedbackApi/v1.schema.json @@ -0,0 +1,90 @@ +{ + "$id": "https://mapcolonies.com/vector/feedbackApi/v1", + "type": "object", + "title": "vectorFeedbackApiV1", + "description": "Mapping configuration for external vector data to OSM tags", + "allOf": [ + { "$ref": "https://mapcolonies.com/common/boilerplate/v2" }, + { "$ref": "#/definitions/databases" }, + { "$ref": "#/definitions/appSchema" } + ], + "definitions": { + "databases": { + "type": "object", + "required": ["redis", "kafka"], + "properties": { + "redis": { + "allOf": [ + { "$ref": "https://mapcolonies.com/common/redis/v2" }, + { + "type": "object", + "properties": { + "connectTimeoutMs": { + "type": "number", + "x-env-value": "REDIS_CONNECT_TIMEOUT", + "default": 5000, + "minimum": 500 + } + }, + "required": ["connectTimeoutMs"] + } + ] + }, + "kafka": { + "allOf": [ + { "$ref": "https://mapcolonies.com/common/kafka/v1" }, + { + "type": "object", + "required": ["outputTopic"] + } + ] + } + } + }, + "appSchema": { + "required": ["application", "schema"], + "type": "object", + "properties": { + "application": { + "type": "object", + "required": ["hashKey", "userValidation"], + "properties": { + "hashKey": { + "type": "object", + "required": ["enabled"], + "properties": { + "enabled": { + "type": "boolean", + "x-env-value": "APP_HASHKEY_ENABLED" + }, + "value": { + "type": "string", + "x-env-value": "APP_HASHKEY_VALUE" + } + }, + "if": { + "properties": { "enabled": { "const": true } } + }, + "then": { + "required": ["value"] + } + }, + "userValidation": { + "type": "array", + "items": { "type": "string" }, + "x-env-value": "USER_ID_DOMAIN", + "x-env-format": "json" + } + } + }, + "schema": { + "type": "object", + "properties": { + "provider": { "type": "string", "x-env-value": "SCHEMA_PROVIDER" }, + "filePath": { "type": "string", "x-env-value": "SCHEMA_FILE_PATH" } + } + } + } + } + } +} From 390e059e2519355bfd698d7a6ffd071d8cb2e9a5 Mon Sep 17 00:00:00 2001 From: Eyal Rozen Date: Thu, 11 Jun 2026 08:53:57 +0300 Subject: [PATCH 03/14] fix: make producer, consumer not required --- schemas/common/kafka/v1.schema.json | 2 +- schemas/vector/feedbackApi/v1.schema.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/schemas/common/kafka/v1.schema.json b/schemas/common/kafka/v1.schema.json index 2bc574bb..5fff2721 100644 --- a/schemas/common/kafka/v1.schema.json +++ b/schemas/common/kafka/v1.schema.json @@ -2,7 +2,7 @@ "$id": "https://mapcolonies.com/common/kafka/v1", "description": "Kafka configuration", "type": "object", - "required": ["brokers", "enableSslAuth", "sslPaths", "kafkaConsumer", "kafkaProducer"], + "required": ["brokers", "enableSslAuth", "sslPaths"], "unevaluatedProperties": true, "properties": { "brokers": { diff --git a/schemas/vector/feedbackApi/v1.schema.json b/schemas/vector/feedbackApi/v1.schema.json index 3c28be3a..611b3186 100644 --- a/schemas/vector/feedbackApi/v1.schema.json +++ b/schemas/vector/feedbackApi/v1.schema.json @@ -35,7 +35,7 @@ { "$ref": "https://mapcolonies.com/common/kafka/v1" }, { "type": "object", - "required": ["outputTopic"] + "required": ["kafkaConsumer", "kafkaProducer", "outputTopic"] } ] } From e7346d568a410912bc43c395145bb4b6077c0ac9 Mon Sep 17 00:00:00 2001 From: Eyal Rozen Date: Thu, 11 Jun 2026 12:56:04 +0300 Subject: [PATCH 04/14] chore: remove redundant naming --- schemas/common/kafka/v1.schema.json | 4 ++-- schemas/vector/feedbackApi/v1.schema.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/schemas/common/kafka/v1.schema.json b/schemas/common/kafka/v1.schema.json index 5fff2721..9f78b5c2 100644 --- a/schemas/common/kafka/v1.schema.json +++ b/schemas/common/kafka/v1.schema.json @@ -27,7 +27,7 @@ "sasl": { "$ref": "#/definitions/sasl" }, - "kafkaConsumer": { + "consumer": { "type": "object", "properties": { "groupId": { @@ -37,7 +37,7 @@ } } }, - "kafkaProducer": { + "producer": { "type": "object" }, "outputTopic": { diff --git a/schemas/vector/feedbackApi/v1.schema.json b/schemas/vector/feedbackApi/v1.schema.json index 611b3186..cd0d535a 100644 --- a/schemas/vector/feedbackApi/v1.schema.json +++ b/schemas/vector/feedbackApi/v1.schema.json @@ -35,7 +35,7 @@ { "$ref": "https://mapcolonies.com/common/kafka/v1" }, { "type": "object", - "required": ["kafkaConsumer", "kafkaProducer", "outputTopic"] + "required": ["consumer", "producer", "outputTopic"] } ] } From e6a331bc0000b22b1345f3526bf2c4c5449f4bf4 Mon Sep 17 00:00:00 2001 From: Eyal Rozen Date: Mon, 15 Jun 2026 20:21:47 +0300 Subject: [PATCH 05/14] fix: make kafka brokers accept array instead of string --- schemas/common/kafka/v1.schema.json | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/schemas/common/kafka/v1.schema.json b/schemas/common/kafka/v1.schema.json index 9f78b5c2..01641326 100644 --- a/schemas/common/kafka/v1.schema.json +++ b/schemas/common/kafka/v1.schema.json @@ -6,8 +6,18 @@ "unevaluatedProperties": true, "properties": { "brokers": { - "type": "string", - "description": "Comma-separated list of broker addresses", + "oneOf": [ + { + "type": "string", + "description": "Comma-separated list of broker addresses" + }, + { + "type": "array", + "items": { "type": "string" }, + "minItems": 1, + "description": "List of broker addresses" + } + ], "x-env-value": "KAFKA_BROKERS" }, "clientId": { From be87b6da20f6c2997e0657ae71452de4813147a7 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Sun, 21 Jun 2026 11:28:05 +0300 Subject: [PATCH 06/14] chore: commit for workflow From f06f1be1b9d39c6e9559b12d0b30d6cffa671fbe Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Sun, 21 Jun 2026 12:54:06 +0300 Subject: [PATCH 07/14] feat: add redis ttl --- schemas/vector/feedbackApi/v1.schema.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/schemas/vector/feedbackApi/v1.schema.json b/schemas/vector/feedbackApi/v1.schema.json index cd0d535a..ab57f364 100644 --- a/schemas/vector/feedbackApi/v1.schema.json +++ b/schemas/vector/feedbackApi/v1.schema.json @@ -24,9 +24,14 @@ "x-env-value": "REDIS_CONNECT_TIMEOUT", "default": 5000, "minimum": 500 + }, + "ttl": { + "type": "number", + "description": "Redis ttl in seconds", + "x-env-value": "REDIS_TTL" } }, - "required": ["connectTimeoutMs"] + "required": ["connectTimeoutMs", "ttl"] } ] }, From 4b18199249a1bea128e4fcb121786e6fb1b00477 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Sun, 21 Jun 2026 13:15:46 +0300 Subject: [PATCH 08/14] feat: make brokers an array --- schemas/common/kafka/v1.schema.json | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/schemas/common/kafka/v1.schema.json b/schemas/common/kafka/v1.schema.json index 01641326..fae43201 100644 --- a/schemas/common/kafka/v1.schema.json +++ b/schemas/common/kafka/v1.schema.json @@ -6,18 +6,10 @@ "unevaluatedProperties": true, "properties": { "brokers": { - "oneOf": [ - { - "type": "string", - "description": "Comma-separated list of broker addresses" - }, - { - "type": "array", - "items": { "type": "string" }, - "minItems": 1, - "description": "List of broker addresses" - } - ], + "type": "array", + "items": { "type": "string" }, + "minItems": 1, + "description": "List of broker addresses", "x-env-value": "KAFKA_BROKERS" }, "clientId": { From 2da8809651c1b1256dd11e8789848030d46f5ffd Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Sun, 21 Jun 2026 13:31:22 +0300 Subject: [PATCH 09/14] chore: commit for workflow From beea0d9f64e1fc9d5972ab6bcd7e051be8e6c9f6 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Sun, 21 Jun 2026 14:00:00 +0300 Subject: [PATCH 10/14] feat: make sasl optional --- schemas/common/kafka/v1.schema.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/schemas/common/kafka/v1.schema.json b/schemas/common/kafka/v1.schema.json index fae43201..76130571 100644 --- a/schemas/common/kafka/v1.schema.json +++ b/schemas/common/kafka/v1.schema.json @@ -89,6 +89,12 @@ "type": "object", "description": "SASL authentication configuration", "properties": { + "enabled": { + "type": "boolean", + "description": "Enable SASL authentication", + "default": false, + "x-env-value": "KAFKA_SASL_ENABLED" + }, "mechanism": { "type": "string", "description": "SASL mechanism", From f0c8c54f8f244187a6c3e9336216aa86e2545430 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Sun, 21 Jun 2026 14:37:09 +0300 Subject: [PATCH 11/14] feat: make mechanism an enum --- schemas/common/kafka/v1.schema.json | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/schemas/common/kafka/v1.schema.json b/schemas/common/kafka/v1.schema.json index 76130571..fc781ff3 100644 --- a/schemas/common/kafka/v1.schema.json +++ b/schemas/common/kafka/v1.schema.json @@ -88,15 +88,11 @@ "sasl": { "type": "object", "description": "SASL authentication configuration", + "required": ["mechanism", "username", "password"], "properties": { - "enabled": { - "type": "boolean", - "description": "Enable SASL authentication", - "default": false, - "x-env-value": "KAFKA_SASL_ENABLED" - }, "mechanism": { "type": "string", + "enum": ["plain", "scram-sha-256", "scram-sha-512"], "description": "SASL mechanism", "x-env-value": "KAFKA_SASL_MECHANISM" }, From 8e1c909403843eda7b143a112de3318e98357ac4 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Sun, 21 Jun 2026 15:06:46 +0300 Subject: [PATCH 12/14] feat: stricter sasl enum --- schemas/common/kafka/v1.schema.json | 42 ++++++++++++++++++----------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/schemas/common/kafka/v1.schema.json b/schemas/common/kafka/v1.schema.json index fc781ff3..aeee8681 100644 --- a/schemas/common/kafka/v1.schema.json +++ b/schemas/common/kafka/v1.schema.json @@ -88,25 +88,35 @@ "sasl": { "type": "object", "description": "SASL authentication configuration", - "required": ["mechanism", "username", "password"], - "properties": { - "mechanism": { - "type": "string", - "enum": ["plain", "scram-sha-256", "scram-sha-512"], - "description": "SASL mechanism", - "x-env-value": "KAFKA_SASL_MECHANISM" + "oneOf": [ + { + "type": "object", + "required": ["mechanism", "username", "password"], + "properties": { + "mechanism": { "const": "plain", "x-env-value": "KAFKA_SASL_MECHANISM" }, + "username": { "type": "string", "x-env-value": "KAFKA_SASL_USERNAME" }, + "password": { "type": "string", "x-env-value": "KAFKA_SASL_PASSWORD" } + } }, - "username": { - "type": "string", - "description": "SASL username", - "x-env-value": "KAFKA_SASL_USERNAME" + { + "type": "object", + "required": ["mechanism", "username", "password"], + "properties": { + "mechanism": { "const": "scram-sha-256", "x-env-value": "KAFKA_SASL_MECHANISM" }, + "username": { "type": "string", "x-env-value": "KAFKA_SASL_USERNAME" }, + "password": { "type": "string", "x-env-value": "KAFKA_SASL_PASSWORD" } + } }, - "password": { - "type": "string", - "description": "SASL password", - "x-env-value": "KAFKA_SASL_PASSWORD" + { + "type": "object", + "required": ["mechanism", "username", "password"], + "properties": { + "mechanism": { "const": "scram-sha-512", "x-env-value": "KAFKA_SASL_MECHANISM" }, + "username": { "type": "string", "x-env-value": "KAFKA_SASL_USERNAME" }, + "password": { "type": "string", "x-env-value": "KAFKA_SASL_PASSWORD" } + } } - } + ] } } } From 615955797480181259a58de92e39e93321072987 Mon Sep 17 00:00:00 2001 From: Eyal Rozen Date: Wed, 24 Jun 2026 18:03:51 +0300 Subject: [PATCH 13/14] fix: add type to ssl --- schemas/common/kafka/v1.schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/schemas/common/kafka/v1.schema.json b/schemas/common/kafka/v1.schema.json index aeee8681..f29df0cf 100644 --- a/schemas/common/kafka/v1.schema.json +++ b/schemas/common/kafka/v1.schema.json @@ -58,6 +58,7 @@ "then": { "properties": { "sslPaths": { + "type": "object", "required": ["ca", "key", "cert"] } } From 049f3f7ec497a594dadf70e47156091b9c1a276d Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Sun, 28 Jun 2026 11:27:52 +0300 Subject: [PATCH 14/14] feat: add brokers format --- schemas/common/kafka/v1.schema.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schemas/common/kafka/v1.schema.json b/schemas/common/kafka/v1.schema.json index f29df0cf..1f77a6d4 100644 --- a/schemas/common/kafka/v1.schema.json +++ b/schemas/common/kafka/v1.schema.json @@ -10,7 +10,8 @@ "items": { "type": "string" }, "minItems": 1, "description": "List of broker addresses", - "x-env-value": "KAFKA_BROKERS" + "x-env-value": "KAFKA_BROKERS", + "x-env-format": "json" }, "clientId": { "type": "string",