diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 2b3033fc..0975f1e0 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -30,7 +30,7 @@ export interface DeletePayload { } export interface Provider { - streamModelPathsToQueueFile: (modelId: string, pathToTileset: string, productName: string) => Promise; + streamModelPathsToQueueFile: (modelId: string, pathToTileset: string, tilesetFilename: string, productName: string) => Promise; getFile: (filePath: string) => Promise; } diff --git a/src/jobOperations/models/jobOperationsManager.ts b/src/jobOperations/models/jobOperationsManager.ts index 24134d2a..0c048788 100644 --- a/src/jobOperations/models/jobOperationsManager.ts +++ b/src/jobOperations/models/jobOperationsManager.ts @@ -214,6 +214,7 @@ export class JobOperationsManager { const fileCount: number = await this.provider.streamModelPathsToQueueFile( payload.modelId, payload.pathToTileset, + payload.tilesetFilename, payload.metadata.productName! ); this.logger.debug({ @@ -224,6 +225,16 @@ export class JobOperationsManager { }); const tasks = this.createTasks(this.batchSize, payload.modelId); + if (tasks.length === 0) { + this.logger.error({ + msg: 'No tasks were created for the job since no paths were found in the model', + logContext, + modelId: payload.modelId, + modelName: payload.metadata.productName, + }); + throw new Error('No paths were found in the model, no tasks were created for the job'); + } + this.logger.info({ msg: 'Tasks created successfully', logContext, diff --git a/src/providers/baseProvider.ts b/src/providers/baseProvider.ts index 0ebe1756..9f856736 100644 --- a/src/providers/baseProvider.ts +++ b/src/providers/baseProvider.ts @@ -28,61 +28,108 @@ export abstract class BaseProvider implements Prov } @withSpanAsyncV4 - public async streamModelPathsToQueueFile(modelId: string, pathToTileset: string, modelName: string): Promise { + public async streamModelPathsToQueueFile(modelId: string, pathToTileset: string, tilesetFilename: string, modelName: string): Promise { const logContext = { ...this.logContext, function: this.streamModelPathsToQueueFile.name }; - let initialPath = pathToTileset; - if (!initialPath.endsWith(this.crawlingExtension)) { - initialPath = Path.join(initialPath, `tileset${this.crawlingExtension}`); - - initialPath = initialPath.replace(/\\/g, '/').replace(/^\//, ''); - } + let fullPath: string = Path.join(pathToTileset, tilesetFilename); + fullPath = fullPath.replace(/\\/g, '/').replace(/^\//, ''); this.logger.info({ msg: 'Started streaming model paths to queue file', logContext, modelName, modelId, - pathToTileset: initialPath, + pathToTileset: fullPath, }); const visitedFiles = new Set(); - const processingQueue: string[] = [initialPath]; + const processingQueue: string[] = [fullPath]; let totalFilesAdded = 0; while (processingQueue.length > 0) { const currentPath = processingQueue.shift(); if (currentPath === undefined) { + this.logger.debug({ + msg: 'Skipping undefined currentPath', + logContext, + modelId, + path: currentPath, + }); continue; } if (visitedFiles.has(currentPath)) { + this.logger.debug({ + msg: 'Skipping already visited file', + logContext, + modelId, + path: currentPath, + }); continue; } visitedFiles.add(currentPath); + this.logger.debug({ + msg: 'Processing model file', + logContext, + modelId, + path: currentPath, + queueRemaining: processingQueue.length, + }); + try { const buffer = await this.getFile(currentPath); await this.queueFileHandler.writeFileNameToQueueFile(modelId, currentPath); totalFilesAdded++; + this.logger.debug({ + msg: 'Added file to queue file', + logContext, + modelId, + path: currentPath, + totalFilesAdded, + }); + if (currentPath.endsWith(this.crawlingExtension)) { const nestedPaths = this.extractPathsFromJson(buffer, currentPath); for (const nestedPath of nestedPaths) { if (visitedFiles.has(nestedPath)) { + this.logger.debug({ + msg: 'Skipping already visited nested path', + logContext, + modelId, + path: nestedPath, + sourcePath: currentPath, + }); continue; } if (nestedPath.endsWith(this.crawlingExtension)) { processingQueue.push(nestedPath); + this.logger.debug({ + msg: 'Queued nested JSON file for processing', + logContext, + modelId, + path: nestedPath, + sourcePath: currentPath, + queueSize: processingQueue.length, + }); } else { await this.queueFileHandler.writeFileNameToQueueFile(modelId, nestedPath); visitedFiles.add(nestedPath); totalFilesAdded++; + this.logger.debug({ + msg: 'Added nested file to queue file', + logContext, + modelId, + path: nestedPath, + sourcePath: currentPath, + totalFilesAdded, + }); } } } @@ -117,18 +164,43 @@ export abstract class BaseProvider implements Prov } private extractPathsFromJson(buffer: Buffer, currentPath: string): string[] { + const logContext = { ...this.logContext, function: this.extractPathsFromJson.name }; + + this.logger.debug({ + msg: 'Extracting paths from JSON content', + logContext: logContext, + path: currentPath, + nestedJsonPath: this.config.nestedJsonPath, + }); + try { const fileContent = buffer.toString(); const json = JSON.parse(fileContent) as object; const nestedJsonPath = this.config.nestedJsonPath; const results = jsonpath.query(json, nestedJsonPath) as string[]; + this.logger.debug({ + msg: 'Found raw nested path references in JSON', + logContext: logContext, + path: currentPath, + rawPathsCount: results.length, + }); + const dirname = Path.dirname(currentPath); - return results.map((child) => { + const resolvedPaths = results.map((child) => { const joinedPath = dirname === '.' ? child : Path.join(dirname, child); return joinedPath.replace(/\\/g, '/').replace(/^\//, ''); }); + + this.logger.debug({ + msg: 'Resolved nested paths relative to current file', + logContext: logContext, + path: currentPath, + resolvedPathsCount: resolvedPaths.length, + }); + + return resolvedPaths; } catch (err) { this.logger.error({ msg: 'Failed to parse JSON', path: currentPath, err }); return []; diff --git a/tests/integration/providers/baseProvider.spec.ts b/tests/integration/providers/baseProvider.spec.ts index fec9f863..42babc51 100644 --- a/tests/integration/providers/baseProvider.spec.ts +++ b/tests/integration/providers/baseProvider.spec.ts @@ -66,7 +66,8 @@ describe('Crawling tests', () => { }; const json1 = { root: { content: { uri: 'bla/c.b3dm' }, children: [{ content: { url: '2.json' } }] } }; const json2 = {}; - const pathToTileset = '/x/y/0.json'; + const pathToTileset = 'x/y'; + const tilesetFilename = '0.json'; it('should returns all the files', async () => { const modelName = faker.word.sample(); @@ -74,8 +75,8 @@ describe('Crawling tests', () => { const getFileSpy = jest.spyOn(crawler, 'getFile'); - // eslint-disable-next-line @typescript-eslint/require-await getFileSpy.mockImplementation(async (path) => { + await Promise.resolve(); const normalizedPath = path.replace(/\\/g, '/').replace(/^\//, ''); if (normalizedPath === 'x/y/0.json') { @@ -91,7 +92,7 @@ describe('Crawling tests', () => { }); await queueFileHandler.createQueueFile(modelId); - const total = await crawler.streamModelPathsToQueueFile(modelId, pathToTileset, modelName); + const total = await crawler.streamModelPathsToQueueFile(modelId, pathToTileset, tilesetFilename, modelName); const result = fs.readFileSync(`${queueFilePath}/${modelId}`, 'utf-8').trim().split('\n'); @@ -114,7 +115,7 @@ describe('Crawling tests', () => { .spyOn(crawler, 'getFile') .mockRejectedValueOnce(new AppError(StatusCodes.INTERNAL_SERVER_ERROR, 'Internal error', false)); - await expect(crawler.streamModelPathsToQueueFile(modelId, pathToTileset, modelName)).rejects.toThrow(AppError); + await expect(crawler.streamModelPathsToQueueFile(modelId, pathToTileset, tilesetFilename, modelName)).rejects.toThrow(AppError); getFileSpy.mockRestore(); }); @@ -122,7 +123,7 @@ describe('Crawling tests', () => { it('should throw on NOT_FOUND when ignoreNotFound is false', async () => { const getFileSpy = jest.spyOn(crawler, 'getFile').mockRejectedValueOnce(new AppError(StatusCodes.NOT_FOUND, 'Not Found', false)); - await expect(crawler.streamModelPathsToQueueFile(modelId, pathToTileset, modelName)).rejects.toThrow(AppError); + await expect(crawler.streamModelPathsToQueueFile(modelId, pathToTileset, tilesetFilename, modelName)).rejects.toThrow(AppError); getFileSpy.mockRestore(); }); @@ -131,7 +132,7 @@ describe('Crawling tests', () => { const ignoringCrawler = createCrawler({ ignoreNotFound: true }); const getFileSpy = jest.spyOn(ignoringCrawler, 'getFile').mockRejectedValue(new AppError(StatusCodes.NOT_FOUND, 'Not Found', false)); - await expect(ignoringCrawler.streamModelPathsToQueueFile(modelId, pathToTileset, modelName)).resolves.toBe(0); + await expect(ignoringCrawler.streamModelPathsToQueueFile(modelId, pathToTileset, tilesetFilename, modelName)).resolves.toBe(0); getFileSpy.mockRestore(); }); diff --git a/tests/integration/providers/nfsProvider.spec.ts b/tests/integration/providers/nfsProvider.spec.ts index 29ba1432..10d60969 100644 --- a/tests/integration/providers/nfsProvider.spec.ts +++ b/tests/integration/providers/nfsProvider.spec.ts @@ -11,7 +11,7 @@ import { NFSProvider } from '../../../src/providers/nfsProvider'; import { SERVICES } from '../../../src/common/constants'; import { BaseProviderConfig, NFSConfig } from '../../../src/common/interfaces'; import { AppError } from '../../../src/common/appError'; -import { createFile, queueFileHandlerMock } from '../../helpers/mockCreator'; +import { queueFileHandlerMock } from '../../helpers/mockCreator'; import { QueueFileHandler } from '../../../src/handlers/queueFileHandler'; import { NFSHelper } from '../../helpers/nfsHelper'; @@ -62,7 +62,8 @@ describe('NFSProvider tests', () => { const modelId = faker.string.uuid(); const modelName = 'interconnect'; const entryFile = 'tileset.json'; - const pathToTileset = `${modelName}/${entryFile}`; + const pathToTileset = modelName; + const tilesetPath = `${modelName}/${entryFile}`; await queueFileHandler.createQueueFile(modelId); @@ -76,28 +77,29 @@ describe('NFSProvider tests', () => { }, }); - await nfsHelper.createFileOfModel('', pathToTileset, tilesetContent); + await nfsHelper.createFileOfModel('', tilesetPath, tilesetContent); await nfsHelper.createFileOfModel(modelName, textureFile, 'data'); await nfsHelper.createFileOfModel(modelName, childTileset, JSON.stringify({ asset: { version: '1.0' } })); - await provider.streamModelPathsToQueueFile(modelId, pathToTileset, modelName); + await provider.streamModelPathsToQueueFile(modelId, pathToTileset, entryFile, modelName); const result = fs.readFileSync(`${queueFilePath}/${modelId}`, 'utf-8'); - expect(result).toContain(pathToTileset); + expect(result).toContain(tilesetPath); await queueFileHandler.deleteQueueFile(modelId); }); it('if model does not exists in the agreed folder, throws error', async () => { const pathToTileset = faker.word.sample(); + const tilesetFilename = 'tileset.json'; const modelName = faker.word.sample(); const modelId = faker.string.uuid(); (provider as unknown as { config: BaseProviderConfig }).config.ignoreNotFound = false; const result = async () => { - await provider.streamModelPathsToQueueFile(modelId, pathToTileset, modelName); + await provider.streamModelPathsToQueueFile(modelId, pathToTileset, tilesetFilename, modelName); }; await expect(result).rejects.toThrow(AppError); @@ -114,14 +116,14 @@ describe('NFSProvider tests', () => { }); provider = container.resolve(NFSProvider); const pathToTileset = faker.word.sample(); + const tilesetFilename = 'tileset.json'; const modelName = faker.word.sample(); const modelId = faker.string.uuid(); - const file = createFile(); - await nfsHelper.createFileOfModel(pathToTileset, file); + await nfsHelper.createFileOfModel(pathToTileset, tilesetFilename, JSON.stringify({})); queueFileHandlerMock.writeFileNameToQueueFile.mockRejectedValue(new AppError(httpStatus.INTERNAL_SERVER_ERROR, 'queueFileHandler', false)); const result = async () => { - await provider.streamModelPathsToQueueFile(modelId, pathToTileset, modelName); + await provider.streamModelPathsToQueueFile(modelId, pathToTileset, tilesetFilename, modelName); }; await expect(result).rejects.toThrow(AppError); diff --git a/tests/integration/providers/s3Provider.spec.ts b/tests/integration/providers/s3Provider.spec.ts index 2c4ce607..d8301b34 100644 --- a/tests/integration/providers/s3Provider.spec.ts +++ b/tests/integration/providers/s3Provider.spec.ts @@ -103,7 +103,7 @@ describe('S3Provider tests', () => { await queueFileHandler.createQueueFile(modelId); - const totalAdded = await provider.streamModelPathsToQueueFile(modelId, rootTileset, modelName); + const totalAdded = await provider.streamModelPathsToQueueFile(modelId, '', rootTileset, modelName); const result = fs.readFileSync(`${queueFilePath}/${modelId}`, 'utf-8'); const filesInQueue = result @@ -126,9 +126,10 @@ describe('S3Provider tests', () => { await queueFileHandler.createQueueFile(modelId); const modelName = faker.word.sample(); const pathToTileset = faker.word.sample(); + const tilesetFilename = 'tileset.json'; const result = async () => { - await provider.streamModelPathsToQueueFile(modelId, pathToTileset, modelName); + await provider.streamModelPathsToQueueFile(modelId, pathToTileset, tilesetFilename, modelName); }; await expect(result).rejects.toThrow(AppError); diff --git a/tests/unit/jobOperations/models/jobOperationsManager.spec.ts b/tests/unit/jobOperations/models/jobOperationsManager.spec.ts index 936b0efa..ce97196d 100644 --- a/tests/unit/jobOperations/models/jobOperationsManager.spec.ts +++ b/tests/unit/jobOperations/models/jobOperationsManager.spec.ts @@ -104,6 +104,24 @@ describe('jobOperationsManager', () => { //Assert expect(response).toBeUndefined(); + expect(configProviderMock.streamModelPathsToQueueFile).toHaveBeenCalledWith( + payload.modelId, + payload.pathToTileset, + payload.tilesetFilename, + payload.metadata.productName + ); + }); + + it('rejects when no paths were found in the model', async () => { + const jobId = faker.string.uuid(); + queueFileHandlerMock.createQueueFile.mockResolvedValue(undefined); + configProviderMock.streamModelPathsToQueueFile.mockResolvedValue(0); + queueFileHandlerMock.readline.mockReturnValue(null); + queueFileHandlerMock.deleteQueueFile.mockResolvedValue(undefined); + + await expect(jobOperationsManager.createModel(payload, jobId)).rejects.toThrow( + 'No paths were found in the model, no tasks were created for the job' + ); }); it(`rejects if couldn't createQueueFile queue file`, async () => {