fix: fixed tilesetFilename added to the path (MAPCO-10817)

src/common/interfaces.ts

@@ -30,7 +30,7 @@ export interface DeletePayload {
 }
 
 export interface Provider {
-  streamModelPathsToQueueFile: (modelId: string, pathToTileset: string, productName: string) => Promise<number>;
+  streamModelPathsToQueueFile: (modelId: string, pathToTileset: string, tilesetFilename: string, productName: string) => Promise<number>;
   getFile: (filePath: string) => Promise<Buffer>;
 }
 

src/jobOperations/models/jobOperationsManager.ts

@@ -214,6 +214,7 @@ export class JobOperationsManager {
       const fileCount: number = await this.provider.streamModelPathsToQueueFile(
         payload.modelId,
         payload.pathToTileset,
+        payload.tilesetFilename,
         payload.metadata.productName!
       );
       this.logger.debug({
@@ -224,6 +225,16 @@ export class JobOperationsManager {
       });
 
       const tasks = this.createTasks(this.batchSize, payload.modelId);
+      if (tasks.length === 0) {
+        this.logger.error({
+          msg: 'No tasks were created for the job since no paths were found in the model',
+          logContext,
+          modelId: payload.modelId,
+          modelName: payload.metadata.productName,
+        });
+        throw new Error('No paths were found in the model, no tasks were created for the job');
+      }
+
       this.logger.info({
         msg: 'Tasks created successfully',
         logContext,

src/providers/baseProvider.ts

@@ -28,61 +28,108 @@ export abstract class BaseProvider<T extends BaseProviderConfig> implements Prov
   }
 
   @withSpanAsyncV4
-  public async streamModelPathsToQueueFile(modelId: string, pathToTileset: string, modelName: string): Promise<number> {
+  public async streamModelPathsToQueueFile(modelId: string, pathToTileset: string, tilesetFilename: string, modelName: string): Promise<number> {
     const logContext = { ...this.logContext, function: this.streamModelPathsToQueueFile.name };
 
-    let initialPath = pathToTileset;
-    if (!initialPath.endsWith(this.crawlingExtension)) {
-      initialPath = Path.join(initialPath, `tileset${this.crawlingExtension}`);
-
-      initialPath = initialPath.replace(/\\/g, '/').replace(/^\//, '');
-    }
+    let fullPath: string = Path.join(pathToTileset, tilesetFilename);
+    fullPath = fullPath.replace(/\\/g, '/').replace(/^\//, '');
 
     this.logger.info({
       msg: 'Started streaming model paths to queue file',
       logContext,
       modelName,
       modelId,
-      pathToTileset: initialPath,
+      pathToTileset: fullPath,
     });
 
     const visitedFiles = new Set<string>();
-    const processingQueue: string[] = [initialPath];
+    const processingQueue: string[] = [fullPath];
     let totalFilesAdded = 0;
 
     while (processingQueue.length > 0) {
       const currentPath = processingQueue.shift();
 
       if (currentPath === undefined) {
+        this.logger.debug({
+          msg: 'Skipping undefined currentPath',
+          logContext,
+          modelId,
+          path: currentPath,
+        });
         continue;
       }
 
       if (visitedFiles.has(currentPath)) {
+        this.logger.debug({
+          msg: 'Skipping already visited file',
+          logContext,
+          modelId,
+          path: currentPath,
+        });
         continue;
       }
 
       visitedFiles.add(currentPath);
 
+      this.logger.debug({
+        msg: 'Processing model file',
+        logContext,
+        modelId,
+        path: currentPath,
+        queueRemaining: processingQueue.length,
+      });
+
       try {
         const buffer = await this.getFile(currentPath);
 
         await this.queueFileHandler.writeFileNameToQueueFile(modelId, currentPath);
         totalFilesAdded++;
 
+        this.logger.debug({
+          msg: 'Added file to queue file',
+          logContext,
+          modelId,
+          path: currentPath,
+          totalFilesAdded,
+        });
+
         if (currentPath.endsWith(this.crawlingExtension)) {
           const nestedPaths = this.extractPathsFromJson(buffer, currentPath);
 
           for (const nestedPath of nestedPaths) {
             if (visitedFiles.has(nestedPath)) {
+              this.logger.debug({
+                msg: 'Skipping already visited nested path',
+                logContext,
+                modelId,
+                path: nestedPath,
+                sourcePath: currentPath,
+              });
               continue;
             }
 
             if (nestedPath.endsWith(this.crawlingExtension)) {
               processingQueue.push(nestedPath);
+              this.logger.debug({
+                msg: 'Queued nested JSON file for processing',
+                logContext,
+                modelId,
+                path: nestedPath,
+                sourcePath: currentPath,
+                queueSize: processingQueue.length,
+              });
             } else {
               await this.queueFileHandler.writeFileNameToQueueFile(modelId, nestedPath);
               visitedFiles.add(nestedPath);
               totalFilesAdded++;
+              this.logger.debug({
+                msg: 'Added nested file to queue file',
+                logContext,
+                modelId,
+                path: nestedPath,
+                sourcePath: currentPath,
+                totalFilesAdded,
+              });
             }
           }
         }
@@ -117,18 +164,43 @@ export abstract class BaseProvider<T extends BaseProviderConfig> implements Prov
   }
 
   private extractPathsFromJson(buffer: Buffer, currentPath: string): string[] {
+    const logContext = { ...this.logContext, function: this.extractPathsFromJson.name };
+
+    this.logger.debug({
+      msg: 'Extracting paths from JSON content',
+      logContext: logContext,
+      path: currentPath,
+      nestedJsonPath: this.config.nestedJsonPath,
+    });
+
     try {
       const fileContent = buffer.toString();
       const json = JSON.parse(fileContent) as object;
       const nestedJsonPath = this.config.nestedJsonPath;
       const results = jsonpath.query(json, nestedJsonPath) as string[];
 
+      this.logger.debug({
+        msg: 'Found raw nested path references in JSON',
+        logContext: logContext,
+        path: currentPath,
+        rawPathsCount: results.length,
+      });
+
       const dirname = Path.dirname(currentPath);
 
-      return results.map((child) => {
+      const resolvedPaths = results.map((child) => {
         const joinedPath = dirname === '.' ? child : Path.join(dirname, child);
         return joinedPath.replace(/\\/g, '/').replace(/^\//, '');
       });
+
+      this.logger.debug({
+        msg: 'Resolved nested paths relative to current file',
+        logContext: logContext,
+        path: currentPath,
+        resolvedPathsCount: resolvedPaths.length,
+      });
+
+      return resolvedPaths;
     } catch (err) {
       this.logger.error({ msg: 'Failed to parse JSON', path: currentPath, err });
       return [];

tests/integration/providers/baseProvider.spec.ts

@@ -66,16 +66,17 @@ describe('Crawling tests', () => {
     };
     const json1 = { root: { content: { uri: 'bla/c.b3dm' }, children: [{ content: { url: '2.json' } }] } };
     const json2 = {};
-    const pathToTileset = '/x/y/0.json';
+    const pathToTileset = 'x/y';
+    const tilesetFilename = '0.json';
 
     it('should returns all the files', async () => {
       const modelName = faker.word.sample();
       const modelId = faker.string.uuid();
 
       const getFileSpy = jest.spyOn(crawler, 'getFile');
 
-      // eslint-disable-next-line @typescript-eslint/require-await
       getFileSpy.mockImplementation(async (path) => {
+        await Promise.resolve();
         const normalizedPath = path.replace(/\\/g, '/').replace(/^\//, '');
 
         if (normalizedPath === 'x/y/0.json') {
@@ -91,7 +92,7 @@ describe('Crawling tests', () => {
       });
 
       await queueFileHandler.createQueueFile(modelId);
-      const total = await crawler.streamModelPathsToQueueFile(modelId, pathToTileset, modelName);
+      const total = await crawler.streamModelPathsToQueueFile(modelId, pathToTileset, tilesetFilename, modelName);
 
       const result = fs.readFileSync(`${queueFilePath}/${modelId}`, 'utf-8').trim().split('\n');
 
@@ -114,15 +115,15 @@ describe('Crawling tests', () => {
           .spyOn(crawler, 'getFile')
           .mockRejectedValueOnce(new AppError(StatusCodes.INTERNAL_SERVER_ERROR, 'Internal error', false));
 
-        await expect(crawler.streamModelPathsToQueueFile(modelId, pathToTileset, modelName)).rejects.toThrow(AppError);
+        await expect(crawler.streamModelPathsToQueueFile(modelId, pathToTileset, tilesetFilename, modelName)).rejects.toThrow(AppError);
 
         getFileSpy.mockRestore();
       });
 
       it('should throw on NOT_FOUND when ignoreNotFound is false', async () => {
         const getFileSpy = jest.spyOn(crawler, 'getFile').mockRejectedValueOnce(new AppError(StatusCodes.NOT_FOUND, 'Not Found', false));
 
-        await expect(crawler.streamModelPathsToQueueFile(modelId, pathToTileset, modelName)).rejects.toThrow(AppError);
+        await expect(crawler.streamModelPathsToQueueFile(modelId, pathToTileset, tilesetFilename, modelName)).rejects.toThrow(AppError);
 
         getFileSpy.mockRestore();
       });
@@ -131,7 +132,7 @@ describe('Crawling tests', () => {
         const ignoringCrawler = createCrawler({ ignoreNotFound: true });
         const getFileSpy = jest.spyOn(ignoringCrawler, 'getFile').mockRejectedValue(new AppError(StatusCodes.NOT_FOUND, 'Not Found', false));
 
-        await expect(ignoringCrawler.streamModelPathsToQueueFile(modelId, pathToTileset, modelName)).resolves.toBe(0);
+        await expect(ignoringCrawler.streamModelPathsToQueueFile(modelId, pathToTileset, tilesetFilename, modelName)).resolves.toBe(0);
 
         getFileSpy.mockRestore();
       });

tests/integration/providers/nfsProvider.spec.ts

@@ -11,7 +11,7 @@ import { NFSProvider } from '../../../src/providers/nfsProvider';
 import { SERVICES } from '../../../src/common/constants';
 import { BaseProviderConfig, NFSConfig } from '../../../src/common/interfaces';
 import { AppError } from '../../../src/common/appError';
-import { createFile, queueFileHandlerMock } from '../../helpers/mockCreator';
+import { queueFileHandlerMock } from '../../helpers/mockCreator';
 import { QueueFileHandler } from '../../../src/handlers/queueFileHandler';
 import { NFSHelper } from '../../helpers/nfsHelper';
 
@@ -62,7 +62,8 @@ describe('NFSProvider tests', () => {
       const modelId = faker.string.uuid();
       const modelName = 'interconnect';
       const entryFile = 'tileset.json';
-      const pathToTileset = `${modelName}/${entryFile}`;
+      const pathToTileset = modelName;
+      const tilesetPath = `${modelName}/${entryFile}`;
 
       await queueFileHandler.createQueueFile(modelId);
 
@@ -76,28 +77,29 @@ describe('NFSProvider tests', () => {
         },
       });
 
-      await nfsHelper.createFileOfModel('', pathToTileset, tilesetContent);
+      await nfsHelper.createFileOfModel('', tilesetPath, tilesetContent);
 
       await nfsHelper.createFileOfModel(modelName, textureFile, 'data');
       await nfsHelper.createFileOfModel(modelName, childTileset, JSON.stringify({ asset: { version: '1.0' } }));
 
-      await provider.streamModelPathsToQueueFile(modelId, pathToTileset, modelName);
+      await provider.streamModelPathsToQueueFile(modelId, pathToTileset, entryFile, modelName);
 
       const result = fs.readFileSync(`${queueFilePath}/${modelId}`, 'utf-8');
 
-      expect(result).toContain(pathToTileset);
+      expect(result).toContain(tilesetPath);
       await queueFileHandler.deleteQueueFile(modelId);
     });
 
     it('if model does not exists in the agreed folder, throws error', async () => {
       const pathToTileset = faker.word.sample();
+      const tilesetFilename = 'tileset.json';
       const modelName = faker.word.sample();
       const modelId = faker.string.uuid();
 
       (provider as unknown as { config: BaseProviderConfig }).config.ignoreNotFound = false;
 
       const result = async () => {
-        await provider.streamModelPathsToQueueFile(modelId, pathToTileset, modelName);
+        await provider.streamModelPathsToQueueFile(modelId, pathToTileset, tilesetFilename, modelName);
       };
 
       await expect(result).rejects.toThrow(AppError);
@@ -114,14 +116,14 @@ describe('NFSProvider tests', () => {
       });
       provider = container.resolve(NFSProvider);
       const pathToTileset = faker.word.sample();
+      const tilesetFilename = 'tileset.json';
       const modelName = faker.word.sample();
       const modelId = faker.string.uuid();
-      const file = createFile();
-      await nfsHelper.createFileOfModel(pathToTileset, file);
+      await nfsHelper.createFileOfModel(pathToTileset, tilesetFilename, JSON.stringify({}));
       queueFileHandlerMock.writeFileNameToQueueFile.mockRejectedValue(new AppError(httpStatus.INTERNAL_SERVER_ERROR, 'queueFileHandler', false));
 
       const result = async () => {
-        await provider.streamModelPathsToQueueFile(modelId, pathToTileset, modelName);
+        await provider.streamModelPathsToQueueFile(modelId, pathToTileset, tilesetFilename, modelName);
       };
 
       await expect(result).rejects.toThrow(AppError);

tests/integration/providers/s3Provider.spec.ts

@@ -103,7 +103,7 @@ describe('S3Provider tests', () => {
 
       await queueFileHandler.createQueueFile(modelId);
 
-      const totalAdded = await provider.streamModelPathsToQueueFile(modelId, rootTileset, modelName);
+      const totalAdded = await provider.streamModelPathsToQueueFile(modelId, '', rootTileset, modelName);
 
       const result = fs.readFileSync(`${queueFilePath}/${modelId}`, 'utf-8');
       const filesInQueue = result
@@ -126,9 +126,10 @@ describe('S3Provider tests', () => {
       await queueFileHandler.createQueueFile(modelId);
       const modelName = faker.word.sample();
       const pathToTileset = faker.word.sample();
+      const tilesetFilename = 'tileset.json';
 
       const result = async () => {
-        await provider.streamModelPathsToQueueFile(modelId, pathToTileset, modelName);
+        await provider.streamModelPathsToQueueFile(modelId, pathToTileset, tilesetFilename, modelName);
       };
 
       await expect(result).rejects.toThrow(AppError);

tests/unit/jobOperations/models/jobOperationsManager.spec.ts

@@ -104,6 +104,24 @@ describe('jobOperationsManager', () => {
 
       //Assert
       expect(response).toBeUndefined();
+      expect(configProviderMock.streamModelPathsToQueueFile).toHaveBeenCalledWith(
+        payload.modelId,
+        payload.pathToTileset,
+        payload.tilesetFilename,
+        payload.metadata.productName
+      );
+    });
+
+    it('rejects when no paths were found in the model', async () => {
+      const jobId = faker.string.uuid();
+      queueFileHandlerMock.createQueueFile.mockResolvedValue(undefined);
+      configProviderMock.streamModelPathsToQueueFile.mockResolvedValue(0);
+      queueFileHandlerMock.readline.mockReturnValue(null);
+      queueFileHandlerMock.deleteQueueFile.mockResolvedValue(undefined);
+
+      await expect(jobOperationsManager.createModel(payload, jobId)).rejects.toThrow(
+        'No paths were found in the model, no tasks were created for the job'
+      );
     });
 
     it(`rejects if couldn't createQueueFile queue file`, async () => {