From 79ecb8ea9ba7b6f331f2fd3a1c20d6b9e323f6cf Mon Sep 17 00:00:00 2001 From: yevhenii-moroziuk Date: Fri, 21 Feb 2025 19:40:31 +0200 Subject: [PATCH] HCK-10091: Allow RE --- api/re.js | 3 +- package-lock.json | 5 +- reverse_engineering/api.js | 145 +---------------------------- reverse_engineering/reFromFile.js | 146 ++++++++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 147 deletions(-) create mode 100644 reverse_engineering/reFromFile.js diff --git a/api/re.js b/api/re.js index 7b649bdb..ebd678e2 100644 --- a/api/re.js +++ b/api/re.js @@ -1,3 +1,4 @@ const { adaptJsonSchema } = require('../reverse_engineering/adaptJsonSchema'); +const { reFromFile } = require('../reverse_engineering/reFromFile'); -module.exports = { adaptJsonSchema }; +module.exports = { adaptJsonSchema, reFromFile }; diff --git a/package-lock.json b/package-lock.json index 6c5037a8..ef9b950e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,13 @@ { "name": "Avro", - "version": "0.2.8", + "version": "0.2.10", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "Avro", - "version": "0.2.8", + "version": "0.2.10", + "hasInstallScript": true, "dependencies": { "avsc": "5.4.6", "esbuild-plugin-copy": "2.1.1", diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js index 20088fe4..a6def4cb 100644 --- a/reverse_engineering/api.js +++ b/reverse_engineering/api.js @@ -1,147 +1,4 @@ -const _ = require('lodash'); const { adaptJsonSchema } = require('./adaptJsonSchema'); -const { initPluginConfiguration } = require('../shared/customProperties'); -const mapJsonSchema = require('../shared/mapJsonSchema'); -const convertToJsonSchemas = require('./helpers/convertToJsonSchemas'); -const { openAvroFile } = require('./helpers/fileHelper'); -const { getNamespace, handleErrorObject } = require('./helpers/generalHelper'); - -const reFromFile = async (data, logger, callback, app) => { - initPluginConfiguration(data.pluginConfiguration, logger); - - try { - const { filePath } = data; - const avroSchema = await openAvroFile(filePath); - const jsonSchemas = convertToJsonSchemas(avroSchema); - - const { schemaRegistryType, schemaRegistryUrl } = _.first(getSchemasData(avroSchema)); - - return callback( - null, - getPackages(avroSchema, jsonSchemas), - { - schemaRegistryType, - schemaRegistryUrl, - }, - [], - 'multipleSchema', - ); - } catch (err) { - const errorData = handleErrorObject(err); - logger.log('error', errorData, 'Parsing Avro Schema Error'); - - return callback(errorData); - } -}; - -const getPackages = (avroSchema, jsonSchemas) => { - const schemasData = getSchemasData(avroSchema); - const isAvroSchemaSplittedIntoMultipleJsonSchemas = schemasData.length === 1 && jsonSchemas.length > 1; - const singleNamespaceForOneAvroSchemaSplittedIntoMultipleJsonSchemas = schemasData[0]?.namespace; - - return jsonSchemas.map((jsonSchema, index) => { - const { namespace, schemaType, schemaGroupName, confluentSubjectName, schemaTopic, confluentVersion } = - schemasData[index] || {}; - const schemaNamespace = isAvroSchemaSplittedIntoMultipleJsonSchemas - ? singleNamespaceForOneAvroSchemaSplittedIntoMultipleJsonSchemas - : namespace; - - const schemaNameStrategy = inferSchemaNameStrategy({ - name: jsonSchema.title, - namespace, - confluentSubjectName, - schemaTopic, - }); - let references = []; - mapJsonSchema(field => { - if (!field.$ref) { - return field; - } - - const COLLECTION_REFERENCE_PREFIX = '#collection/definitions/'; - const isCollectionRef = field.$ref.startsWith(COLLECTION_REFERENCE_PREFIX); - if (isCollectionRef) { - references = [...references, field.$ref.slice(COLLECTION_REFERENCE_PREFIX.length)]; - } - return field; - })(jsonSchema); - - return { - objectNames: { - collectionName: jsonSchema.title, - }, - doc: { - dbName: schemaNamespace || '', - collectionName: jsonSchema.title, - bucketInfo: { - name: schemaNamespace || '', - }, - }, - jsonSchema: JSON.stringify({ - ...jsonSchema, - schemaType: schemaType, - schemaTopic: schemaTopic, - schemaGroupName: schemaGroupName, - confluentSubjectName: confluentSubjectName, - confluentVersion: confluentVersion, - ...(schemaNameStrategy && { schemaNameStrategy }), - }), - references, - }; - }); -}; - -const inferSchemaNameStrategy = ({ name, namespace, confluentSubjectName, schemaTopic }) => { - let splittedSubjectName = (confluentSubjectName || '').split('-').filter(Boolean); - const endsWithSchemaType = ['key', 'value'].includes(_.last(splittedSubjectName)); - const startsWithTopic = _.first(splittedSubjectName) === schemaTopic && schemaTopic !== namespace + '.' + name; - const startsWithNamespace = namespace && _.first(splittedSubjectName)?.startsWith(namespace + '.'); - - if (startsWithNamespace) { - splittedSubjectName[0] = splittedSubjectName[0].slice(namespace.length + 1); - } - - if (endsWithSchemaType) { - splittedSubjectName = splittedSubjectName.slice(0, -1); - } - - if (startsWithTopic) { - splittedSubjectName = splittedSubjectName.slice(1); - } - - if (startsWithTopic && _.isEmpty(splittedSubjectName)) { - if (name === schemaTopic) { - return 'RecordNameStrategy'; - } - - return 'TopicNameStrategy'; - } - - const splittedRecordName = [...(name || '').split('-')].filter(Boolean); - const recordNameStrategy = _.isEqual(splittedRecordName, splittedSubjectName); - - if (!recordNameStrategy) { - return; - } - - return startsWithTopic ? 'TopicRecordNameStrategy' : 'RecordNameStrategy'; -}; - -const getSchemasData = avroSchema => { - avroSchema = _.isArray(avroSchema) ? avroSchema : [avroSchema]; - - return avroSchema.map(schema => ({ - namespace: getNamespace(schema) || '', - schemaGroupName: schema.schemaGroupName, - schemaRegistryType: schema.schemaRegistryType, - schemaRegistryUrl: schema.schemaRegistryUrl, - confluentSubjectName: schema.confluentSubjectName, - schemaTopic: schema.schemaTopic, - schemaType: schema.schemaType, - ...((typeof schema.version === 'number' || typeof schema.version === 'string') && { - confluentVersion: String(schema.version), - }), - })); -}; +const { reFromFile } = require('./reFromFile'); module.exports = { reFromFile, adaptJsonSchema }; diff --git a/reverse_engineering/reFromFile.js b/reverse_engineering/reFromFile.js new file mode 100644 index 00000000..63fa14b5 --- /dev/null +++ b/reverse_engineering/reFromFile.js @@ -0,0 +1,146 @@ +const _ = require('lodash'); +const { initPluginConfiguration } = require('../shared/customProperties'); +const mapJsonSchema = require('../shared/mapJsonSchema'); +const convertToJsonSchemas = require('../reverse_engineering/helpers/convertToJsonSchemas'); +const { openAvroFile } = require('../reverse_engineering/helpers/fileHelper'); +const { getNamespace, handleErrorObject } = require('../reverse_engineering/helpers/generalHelper'); + +const reFromFile = async (data, logger, callback, app) => { + initPluginConfiguration(data.pluginConfiguration, logger); + + try { + const { filePath } = data; + const avroSchema = await openAvroFile(filePath); + const jsonSchemas = convertToJsonSchemas(avroSchema); + + const { schemaRegistryType, schemaRegistryUrl } = _.first(getSchemasData(avroSchema)); + + return callback( + null, + getPackages(avroSchema, jsonSchemas), + { + schemaRegistryType, + schemaRegistryUrl, + }, + [], + 'multipleSchema', + ); + } catch (err) { + const errorData = handleErrorObject(err); + logger.log('error', errorData, 'Parsing Avro Schema Error'); + + return callback(errorData); + } +}; + +const getPackages = (avroSchema, jsonSchemas) => { + const schemasData = getSchemasData(avroSchema); + const isAvroSchemaSplittedIntoMultipleJsonSchemas = schemasData.length === 1 && jsonSchemas.length > 1; + const singleNamespaceForOneAvroSchemaSplittedIntoMultipleJsonSchemas = schemasData[0]?.namespace; + + return jsonSchemas.map((jsonSchema, index) => { + const { namespace, schemaType, schemaGroupName, confluentSubjectName, schemaTopic, confluentVersion } = + schemasData[index] || {}; + const schemaNamespace = isAvroSchemaSplittedIntoMultipleJsonSchemas + ? singleNamespaceForOneAvroSchemaSplittedIntoMultipleJsonSchemas + : namespace; + + const schemaNameStrategy = inferSchemaNameStrategy({ + name: jsonSchema.title, + namespace, + confluentSubjectName, + schemaTopic, + }); + let references = []; + mapJsonSchema(field => { + if (!field.$ref) { + return field; + } + + const COLLECTION_REFERENCE_PREFIX = '#collection/definitions/'; + const isCollectionRef = field.$ref.startsWith(COLLECTION_REFERENCE_PREFIX); + if (isCollectionRef) { + references = [...references, field.$ref.slice(COLLECTION_REFERENCE_PREFIX.length)]; + } + return field; + })(jsonSchema); + + return { + objectNames: { + collectionName: jsonSchema.title, + }, + doc: { + dbName: schemaNamespace || '', + collectionName: jsonSchema.title, + bucketInfo: { + name: schemaNamespace || '', + }, + }, + jsonSchema: JSON.stringify({ + ...jsonSchema, + schemaType: schemaType, + schemaTopic: schemaTopic, + schemaGroupName: schemaGroupName, + confluentSubjectName: confluentSubjectName, + confluentVersion: confluentVersion, + ...(schemaNameStrategy && { schemaNameStrategy }), + }), + references, + }; + }); +}; + +const inferSchemaNameStrategy = ({ name, namespace, confluentSubjectName, schemaTopic }) => { + let splittedSubjectName = (confluentSubjectName || '').split('-').filter(Boolean); + const endsWithSchemaType = ['key', 'value'].includes(_.last(splittedSubjectName)); + const startsWithTopic = _.first(splittedSubjectName) === schemaTopic && schemaTopic !== namespace + '.' + name; + const startsWithNamespace = namespace && _.first(splittedSubjectName)?.startsWith(namespace + '.'); + + if (startsWithNamespace) { + splittedSubjectName[0] = splittedSubjectName[0].slice(namespace.length + 1); + } + + if (endsWithSchemaType) { + splittedSubjectName = splittedSubjectName.slice(0, -1); + } + + if (startsWithTopic) { + splittedSubjectName = splittedSubjectName.slice(1); + } + + if (startsWithTopic && _.isEmpty(splittedSubjectName)) { + if (name === schemaTopic) { + return 'RecordNameStrategy'; + } + + return 'TopicNameStrategy'; + } + + const splittedRecordName = [...(name || '').split('-')].filter(Boolean); + const recordNameStrategy = _.isEqual(splittedRecordName, splittedSubjectName); + + if (!recordNameStrategy) { + return; + } + + return startsWithTopic ? 'TopicRecordNameStrategy' : 'RecordNameStrategy'; +}; + +const getSchemasData = avroSchema => { + avroSchema = _.isArray(avroSchema) ? avroSchema : [avroSchema]; + + return avroSchema.map(schema => ({ + namespace: getNamespace(schema) || '', + schemaGroupName: schema.schemaGroupName, + schemaRegistryType: schema.schemaRegistryType, + schemaRegistryUrl: schema.schemaRegistryUrl, + confluentSubjectName: schema.confluentSubjectName, + schemaTopic: schema.schemaTopic, + schemaType: schema.schemaType, + ...((typeof schema.version === 'number' || typeof schema.version === 'string') && { + confluentVersion: String(schema.version), + }), + })); +}; + +module.exports = { reFromFile };