feat: add keywords field to ingestion layer metadata (MAPCO-8284)

openapi3.yaml

@@ -459,6 +459,8 @@ components:
           $ref: ./Schema/core/layerMetadata.yaml#/components/schemas/Description
         classification:
           $ref: ./Schema/core/layerMetadata.yaml#/components/schemas/Classification
+        keywords:
+          $ref: ./Schema/core/layerMetadata.yaml#/components/schemas/Keywords
     IngestionNewLayerRequest:
       type: object
       required:
@@ -487,6 +489,8 @@ components:
       properties:
         classification:
           $ref: ./Schema/core/layerMetadata.yaml#/components/schemas/Classification
+        keywords:
+          $ref: ./Schema/core/layerMetadata.yaml#/components/schemas/Keywords
     IngestionUpdateLayerRequest:
       type: object
       required:

src/ingestion/models/ingestionManager.ts

@@ -637,8 +637,13 @@
     const relativeChecksums = this.convertChecksumsToRelativePaths(checksums);
     const taskParameters: IngestionValidationTaskParams = { checksums: relativeChecksums };
 
+    const keywords = isSwapUpdate
+      ? updateLayer.metadata.keywords
+      : this.mergeKeywords(rasterLayerMetadata.keywords, updateLayer.metadata.keywords);
+
     const updateLayerRelative = {
       ...updateLayer,
+      metadata: { ...updateLayer.metadata, keywords },
       ...{
         inputFiles: {
           metadataShapefilePath: updateLayer.inputFiles.metadataShapefilePath.relative,
@@ -671,6 +676,16 @@
     return createJobRequest;
   }
 
+  private mergeKeywords(existing: string | undefined, incoming: string | undefined): string | undefined {
+    const toTokens = (value: string | undefined): string[] =>
+      (value ?? '')
+        .split(',')
+        .map((keyword) => keyword.trim())
+        .filter((keyword) => keyword.length > 0);
+    const merged = [...new Set([...toTokens(existing), ...toTokens(incoming)])];
+    return merged.length > 0 ? merged.join(',') : undefined;
+  }
+
   @withSpanAsyncV4
   private async getChecksum(shapefilePath: string): Promise<IChecksum[]> {
     const checksums = await Promise.all(getShapefileFiles(shapefilePath).map(async (fileName) => this.getFileChecksum(fileName)));

src/ingestion/schemas/layerCatalogSchema.ts

@@ -91,6 +91,7 @@ export const rasterLayerCatalogSchema = z
             message: 'Product bounding box must be of the shape min_x,min_y,max_x,max_y',
           })
           .optional(),
+        keywords: z.string().optional(),
         displayPath: z.string().uuid(),
         transparency: z.nativeEnum(Transparency),
         tileMimeFormat: tilesMimeFormatSchema,

tests/integration/ingestion/ingestion.spec.ts

@@ -148,6 +148,47 @@ describe('Ingestion', () => {
         expect(response.status).toBe(httpStatusCodes.OK);
         expect(response.body).toStrictEqual(expectedResponseBody);
       });
+
+      it('should return 200 status code when metadata contains keywords', async () => {
+        const layerRequest = createNewLayerRequest({
+          inputFiles: validInputFiles.inputFiles,
+          metadata: { keywords: faker.lorem.words({ min: 1, max: 5 }) },
+        });
+        const newLayerName = getMapServingLayerName(layerRequest.metadata.productId, layerRequest.metadata.productType);
+        const findJobsParams = createFindJobsParams({
+          resourceId: layerRequest.metadata.productId,
+          productType: layerRequest.metadata.productType,
+        });
+        const newJobRequest = createNewJobRequest({
+          ingestionNewLayer: layerRequest,
+          checksums: validInputFiles.checksums,
+        });
+        nock(jobManagerURL).post('/jobs/find', matches(findJobsParams)).reply(httpStatusCodes.OK, []);
+        nock(jobManagerURL)
+          .post('/jobs', matches(JSON.parse(JSON.stringify(newJobRequest))))
+          .reply(httpStatusCodes.OK, jobResponse);
+        nock(catalogServiceURL)
+          .post('/records/find', {
+            metadata: {
+              productId: layerRequest.metadata.productId,
+              productType: layerRequest.metadata.productType,
+            },
+          })
+          .reply(httpStatusCodes.OK, []);
+        nock(mapProxyApiServiceUrl)
+          .get(`/layer/${encodeURIComponent(newLayerName)}`)
+          .reply(httpStatusCodes.NOT_FOUND);
+        const expectedResponseBody: ResponseId = {
+          jobId: jobResponse.id,
+          taskId: jobResponse.taskIds[0]!,
+        };
+
+        const response = await requestSender.ingestNewLayer(layerRequest);
+
+        expect(response).toSatisfyApiSpec();
+        expect(response.status).toBe(httpStatusCodes.OK);
+        expect(response.body).toStrictEqual(expectedResponseBody);
+      });
     });
 
     describe('Bad Path', () => {
@@ -525,6 +566,24 @@ describe('Ingestion', () => {
             { metadata: { productSubType: false } }
           ),
         },
+        {
+          testCase: 'keywords in metadata in req body is not a string',
+          badRequest: merge(
+            createNewLayerRequest({
+              inputFiles: validInputFiles.inputFiles,
+            }),
+            { metadata: { keywords: false } }
+          ),
+        },
+        {
+          testCase: 'keywords in metadata in req body is an array instead of a string',
+          badRequest: merge(
+            createNewLayerRequest({
+              inputFiles: validInputFiles.inputFiles,
+            }),
+            { metadata: { keywords: ['aerial', 'satellite'] } }
+          ),
+        },
         {
           testCase: 'ingestionResolution in req body is not set',
           badRequest: createNewLayerRequest({
@@ -847,6 +906,45 @@ describe('Ingestion', () => {
         expect(response.body).toStrictEqual(expectedResponseBody);
       });
 
+      it('should return 200 status code with update request when metadata contains keywords', async () => {
+        const layerRequest = createUpdateLayerRequest({
+          inputFiles: validInputFiles.inputFiles,
+          callbackUrls: undefined,
+          metadata: { keywords: faker.lorem.words({ min: 1, max: 5 }) },
+        });
+        const updatedLayer = createCatalogLayerResponse();
+        const updatedLayerMetadata = updatedLayer.metadata;
+        const updateLayerName = getMapServingLayerName(updatedLayerMetadata.productId, updatedLayerMetadata.productType);
+        const findJobsParams = createFindJobsParams({
+          resourceId: updatedLayerMetadata.productId,
+          productType: updatedLayerMetadata.productType,
+        });
+        const updateJobRequest = createUpdateJobRequest({
+          ingestionUpdateLayer: layerRequest,
+          rasterLayerMetadata: updatedLayerMetadata,
+          checksums: validInputFiles.checksums,
+        });
+
+        nock(jobManagerURL).post('/jobs/find', matches(findJobsParams)).reply(httpStatusCodes.OK, []);
+        nock(jobManagerURL)
+          .post('/jobs', matches(JSON.parse(JSON.stringify(updateJobRequest))))
+          .reply(httpStatusCodes.OK, jobResponse);
+        nock(catalogServiceURL).post('/records/find', { id: updatedLayerMetadata.id }).reply(httpStatusCodes.OK, [updatedLayer]);
+        nock(mapProxyApiServiceUrl)
+          .get(`/layer/${encodeURIComponent(updateLayerName)}`)
+          .reply(httpStatusCodes.OK);
+        const expectedResponseBody: ResponseId = {
+          jobId: jobResponse.id,
+          taskId: jobResponse.taskIds[0]!,
+        };
+
+        const response = await requestSender.updateLayer(updatedLayerMetadata.id, layerRequest);
+
+        expect(response).toSatisfyApiSpec();
+        expect(response.status).toBe(httpStatusCodes.OK);
+        expect(response.body).toStrictEqual(expectedResponseBody);
+      });
+
       it('should return 200 status code with swap update request when product shapefile is polygon', async () => {
         const layerRequest = createUpdateLayerRequest({ inputFiles: validInputFiles.inputFiles });
         const catalogLayerResponse = createCatalogLayerResponse({
@@ -1091,6 +1189,24 @@ describe('Ingestion', () => {
             metadata: { classification: '00' },
           }),
         },
+        {
+          testCase: 'keywords in metadata in req body is not a string',
+          badRequest: merge(
+            createUpdateLayerRequest({
+              inputFiles: validInputFiles.inputFiles,
+            }),
+            { metadata: { keywords: false } }
+          ),
+        },
+        {
+          testCase: 'keywords in metadata in req body is an array instead of a string',
+          badRequest: merge(
+            createUpdateLayerRequest({
+              inputFiles: validInputFiles.inputFiles,
+            }),
+            { metadata: { keywords: ['aerial', 'satellite'] } }
+          ),
+        },
         {
           testCase: 'ingestionResolution in req body is not set',
           badRequest: createUpdateLayerRequest({

tests/mocks/mockFactory.ts

@@ -272,6 +272,7 @@ export const rasterLayerMetadataGenerators: RasterLayerMetadataPropertiesGenerat
   description: (): string => generateHebrewAlphanumeric({ min: 0, max: 100 }),
   producerName: (): string => generateHebrewAlphanumeric({ min: 0, max: 100 }),
   productSubType: (): string => generateHebrewAlphanumeric({ min: 0, max: 100 }),
+  keywords: (): string => faker.lorem.words({ min: 1, max: 5 }).split(' ').join(','),
   scale: (): number => faker.number.int({ min: INGESTION_VALIDATIONS.scale.min, max: INGESTION_VALIDATIONS.scale.max }),
   srs: (): '4326' => '4326',
   srsName: (): 'WGS84GEO' => 'WGS84GEO',
@@ -506,12 +507,7 @@ export const createUpdateJobRequest = (
   const sourceMount = configMock.get<string>('storageExplorer.layerSourceDir');
   const updateJobAction = isSwapUpdate ? swapUpdateJobType : updateJobType;
 
-  const {
-    ingestionResolution,
-    inputFiles,
-    metadata: { classification },
-    callbackUrls,
-  } = ingestionUpdateLayer;
+  const { ingestionResolution, inputFiles, metadata, callbackUrls } = ingestionUpdateLayer;
   const { displayPath, id, productId, productType, productVersion, productName, tileOutputFormat } = rasterLayerMetadata;
 
   return {
@@ -524,9 +520,7 @@ export const createUpdateJobRequest = (
     status: OperationStatus.PENDING,
     parameters: {
       ingestionResolution,
-      metadata: {
-        classification,
-      },
+      metadata,
       inputFiles: {
         gpkgFilesPath: inputFiles.gpkgFilesPath.map((gpkgFilePath) => relative(sourceMount, join(sourceMount, gpkgFilePath))),
         metadataShapefilePath: relative(sourceMount, join(sourceMount, inputFiles.metadataShapefilePath)),

tests/unit/ingestion/models/ingestionManager.spec.ts

@@ -460,6 +460,68 @@ describe('IngestionManager', () => {
       expect(createIngestionJobSpy).toHaveBeenCalledWith(expect.objectContaining({ type: ingestionSwapUpdateJobType }));
     });
 
+    it('should concatenate new keywords onto the existing ones, deduplicated, on a regular update', async () => {
+      const catalogLayerResponse = generateCatalogLayerResponse();
+      catalogLayerResponse.metadata.keywords = 'forest,urban';
+      const layerRequest = generateUpdateLayerRequest();
+      layerRequest.metadata.keywords = 'urban,coast';
+      const createJobResponse: ICreateJobResponse = { id: faker.string.uuid(), taskIds: [faker.string.uuid()] };
+      findByIdSpy.mockResolvedValue([catalogLayerResponse]);
+      mockValidateManager.validateShapefiles.mockResolvedValue(undefined);
+      mockValidateManager.validateGpkgsSources.mockResolvedValue(undefined);
+      mockInfoManager.getGpkgsInformation.mockResolvedValue(undefined);
+      productManager.read.mockResolvedValue(undefined);
+      mockGeoValidator.validate.mockResolvedValue(undefined);
+      existsMapproxySpy.mockResolvedValue(true);
+      findJobsSpy.mockResolvedValue([]);
+      calcualteChecksumSpy.mockResolvedValue(generateChecksum());
+      createIngestionJobSpy.mockResolvedValue(createJobResponse);
+
+      await ingestionManager.updateLayer(catalogLayerResponse.metadata.id, layerRequest);
+
+      expect(createIngestionJobSpy).toHaveBeenCalledWith(
+        expect.objectContaining({
+          type: ingestionUpdateJobType,
+          parameters: expect.objectContaining({ metadata: expect.objectContaining({ keywords: 'forest,urban,coast' }) }),
+        })
+      );
+    });
+
+    it('should overwrite keywords with the new value on a swap update', async () => {
+      const baseCatalogLayerResponse = generateCatalogLayerResponse();
+      const catalogLayerResponse = {
+        ...baseCatalogLayerResponse,
+        metadata: {
+          ...baseCatalogLayerResponse.metadata,
+          productType: ingestionSwapUpdateProductType,
+          productSubType: ingestionSwapUpdateProductSubType,
+          keywords: 'forest,urban',
+        },
+      };
+      const layerRequest = generateUpdateLayerRequest();
+      layerRequest.metadata.keywords = 'urban,coast';
+      const createJobResponse: ICreateJobResponse = { id: faker.string.uuid(), taskIds: [faker.string.uuid()] };
+      findByIdSpy.mockResolvedValue([catalogLayerResponse]);
+      mockValidateManager.validateShapefiles.mockResolvedValue(undefined);
+      mockValidateManager.validateGpkgsSources.mockResolvedValue(undefined);
+      mockInfoManager.getGpkgsInformation.mockResolvedValue(undefined);
+      productManager.read.mockResolvedValue(undefined);
+      mockGeoValidator.validate.mockResolvedValue(undefined);
+      existsMapproxySpy.mockResolvedValue(true);
+      findJobsSpy.mockResolvedValue([]);
+      calcualteChecksumSpy.mockResolvedValue(generateChecksum());
+      createIngestionJobSpy.mockResolvedValue(createJobResponse);
+
+      await ingestionManager.updateLayer(catalogLayerResponse.metadata.id, layerRequest);
+
+      expect(createIngestionJobSpy).toHaveBeenCalledWith(
+        expect.objectContaining({
+          type: ingestionSwapUpdateJobType,
+          parameters: expect.objectContaining({ metadata: expect.objectContaining({ keywords: 'urban,coast' }) }),
+        })
+      );
+    });
+
     it('should throw not found error when there is no layer in catalog', async () => {
       const layerRequest = generateUpdateLayerRequest();
       const catalogLayerResponse = generateCatalogLayerResponse();