diff --git a/apps/backend/src/app/admin/workspace/workspace-coding-job-definition.controller.spec.ts b/apps/backend/src/app/admin/workspace/workspace-coding-job-definition.controller.spec.ts index 47d9ff80..bcb0752d 100644 --- a/apps/backend/src/app/admin/workspace/workspace-coding-job-definition.controller.spec.ts +++ b/apps/backend/src/app/admin/workspace/workspace-coding-job-definition.controller.spec.ts @@ -1,6 +1,32 @@ import { WorkspaceCodingJobDefinitionController } from './workspace-coding-job-definition.controller'; describe('WorkspaceCodingJobDefinitionController', () => { + it('lists job definitions without planned usage by default', async () => { + const jobDefinitionService = { + getJobDefinitions: jest.fn().mockResolvedValue([]) + }; + const controller = new WorkspaceCodingJobDefinitionController(jobDefinitionService as never); + + await expect(controller.getJobDefinitions(5)).resolves.toEqual([]); + + expect(jobDefinitionService.getJobDefinitions).toHaveBeenCalledWith(5, { + includePlannedUsage: false + }); + }); + + it('lists job definitions with planned usage when requested', async () => { + const jobDefinitionService = { + getJobDefinitions: jest.fn().mockResolvedValue([]) + }; + const controller = new WorkspaceCodingJobDefinitionController(jobDefinitionService as never); + + await expect(controller.getJobDefinitions(5, 'true')).resolves.toEqual([]); + + expect(jobDefinitionService.getJobDefinitions).toHaveBeenCalledWith(5, { + includePlannedUsage: true + }); + }); + it('reads job definitions through the workspace-scoped service path', async () => { const jobDefinitionService = { getJobDefinition: jest.fn().mockResolvedValue({ diff --git a/apps/backend/src/app/admin/workspace/workspace-coding-job-definition.controller.ts b/apps/backend/src/app/admin/workspace/workspace-coding-job-definition.controller.ts index ca16b225..f5e3d64d 100644 --- a/apps/backend/src/app/admin/workspace/workspace-coding-job-definition.controller.ts +++ b/apps/backend/src/app/admin/workspace/workspace-coding-job-definition.controller.ts @@ -8,14 +8,16 @@ import { UseGuards, Body, ValidationPipe, - Res + Res, + Query } from '@nestjs/common'; import { ApiOkResponse, ApiParam, ApiTags, ApiBody, - ApiProduces + ApiProduces, + ApiQuery } from '@nestjs/swagger'; import { Response } from 'express'; import { JwtAuthGuard } from '../../auth/jwt-auth.guard'; @@ -198,6 +200,12 @@ export class WorkspaceCodingJobDefinitionController { @UseGuards(JwtAuthGuard, WorkspaceGuard) @ApiTags('coding') @ApiParam({ name: 'workspace_id', type: Number }) + @ApiQuery({ + name: 'includePlannedUsage', + required: false, + type: Boolean, + description: 'Include expensive planned variable usage calculations.' + }) @ApiOkResponse({ description: 'List of job definitions retrieved successfully.', schema: { @@ -242,9 +250,12 @@ export class WorkspaceCodingJobDefinitionController { } }) async getJobDefinitions( - @WorkspaceId() workspace_id: number + @WorkspaceId() workspace_id: number, + @Query('includePlannedUsage') includePlannedUsage?: string ): Promise { - return this.jobDefinitionService.getJobDefinitions(workspace_id); + return this.jobDefinitionService.getJobDefinitions(workspace_id, { + includePlannedUsage: includePlannedUsage === 'true' + }); } @Get(':workspace_id/coding/job-definitions/approved') diff --git a/apps/backend/src/app/database/entities/file_upload.entity.ts b/apps/backend/src/app/database/entities/file_upload.entity.ts index 07b06b4c..974996d8 100755 --- a/apps/backend/src/app/database/entities/file_upload.entity.ts +++ b/apps/backend/src/app/database/entities/file_upload.entity.ts @@ -11,6 +11,8 @@ export interface StructuredFileData { }; } +export const NO_CODING_SCHEME_REF_NORMALIZED = '__NO_CODING_SCHEME_REF__'; + @Entity() @Unique('file_upload_id', ['file_id', 'workspace_id']) class FileUpload { @@ -34,6 +36,14 @@ class FileUpload { @Column({ type: 'varchar' }) file_id!: string; + @Index() + @Column({ type: 'varchar', nullable: true }) + file_id_normalized?: string | null; + + @Index() + @Column({ type: 'varchar', nullable: true }) + coding_scheme_ref_normalized?: string | null; + @Column({ type: 'timestamp' }) created_at: number; diff --git a/apps/backend/src/app/database/services/coding/coding-freshness.service.spec.ts b/apps/backend/src/app/database/services/coding/coding-freshness.service.spec.ts index d8545c17..430b1883 100644 --- a/apps/backend/src/app/database/services/coding/coding-freshness.service.spec.ts +++ b/apps/backend/src/app/database/services/coding/coding-freshness.service.spec.ts @@ -55,7 +55,7 @@ describe('CodingFreshnessService', () => { revision: number ): void => { (connection.query as jest.Mock).mockImplementation((sql: string) => { - if (sql.includes('WITH unit_candidates')) { + if (sql.includes('matching_unit_files')) { return Promise.resolve(unitIds.map(id => ({ id }))); } if (sql.includes('workspace_test_results_revision')) { @@ -306,7 +306,7 @@ describe('CodingFreshnessService', () => { }); expect(connection.query).toHaveBeenCalledWith( - expect.stringContaining('codingschemeref'), + expect.stringContaining('coding_scheme_ref_normalized'), [1, ['SEPARATE_SCHEME']] ); expect(freshnessRepository.upsert).toHaveBeenCalledWith( @@ -342,6 +342,83 @@ describe('CodingFreshnessService', () => { ); }); + it('prefilters unit files by indexed normalized coding scheme refs', async () => { + (connection.query as jest.Mock).mockResolvedValue([]); + + await ( + service as unknown as { + getUnitIdsByCodingSchemeRefs: ( + workspaceId: number, + codingSchemeRefs: string[] + ) => Promise; + } + ).getUnitIdsByCodingSchemeRefs(1, ['schemes\\separate_scheme.vocs']); + + const [sql, params] = (connection.query as jest.Mock).mock.calls[0]; + expect(sql).toContain('matching_unit_files'); + expect(sql).toContain('unit_file.coding_scheme_ref_normalized = candidate.scheme_ref'); + expect(sql).toContain('unit_file.file_id_normalized IS NOT NULL'); + expect(sql).toContain('unit_refs AS'); + expect(sql).toContain('CROSS JOIN LATERAL'); + expect(sql).toContain('matched_unit_ids'); + expect(sql).toContain( + "REGEXP_REPLACE(UPPER(unit.name), '\\.XML$', '', 'i') = unit_refs.unit_ref" + ); + expect(sql).toContain( + "REGEXP_REPLACE(UPPER(COALESCE(unit.alias, '')), '\\.XML$', '', 'i') = unit_refs.unit_ref" + ); + expect(sql).not.toContain('unit_candidates AS'); + expect(params).toEqual([1, ['SCHEMES\\SEPARATE_SCHEME', 'SEPARATE_SCHEME']]); + }); + + it('does not run the legacy regex fallback when all Unit files have normalized lookup state', async () => { + (connection.query as jest.Mock) + .mockResolvedValueOnce([{ id: 10 }]) + .mockResolvedValueOnce([{ hasLegacy: false }]); + + await expect(( + service as unknown as { + getUnitIdsByCodingSchemeRefs: ( + workspaceId: number, + codingSchemeRefs: string[] + ) => Promise; + } + ).getUnitIdsByCodingSchemeRefs(1, ['scheme_a'])).resolves.toEqual([10]); + + expect(connection.query).toHaveBeenCalledTimes(2); + expect( + (connection.query as jest.Mock).mock.calls + .some(([sql]) => String(sql).includes('legacy_matching_unit_files')) + ).toBe(false); + }); + + it('uses candidate-driven index probes for the legacy regex fallback', async () => { + (connection.query as jest.Mock) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([{ hasLegacy: true }]) + .mockResolvedValueOnce([{ id: 11 }]); + + await expect(( + service as unknown as { + getUnitIdsByCodingSchemeRefs: ( + workspaceId: number, + codingSchemeRefs: string[] + ) => Promise; + } + ).getUnitIdsByCodingSchemeRefs(1, ['scheme_a'])).resolves.toEqual([11]); + + expect(connection.query).toHaveBeenCalledTimes(3); + const [sql, params] = (connection.query as jest.Mock).mock.calls[2]; + expect(sql).toContain('legacy_matching_unit_files'); + expect(sql).toContain('CROSS JOIN LATERAL'); + expect(sql).toContain( + "REGEXP_REPLACE(UPPER(unit.name), '\\.XML$', '', 'i') =" + ); + expect(sql).toContain('legacy_matching_unit_files.unit_ref'); + expect(sql).not.toContain('unit_candidates AS'); + expect(params).toEqual([1, ['SCHEME_A']]); + }); + it('marks coding scheme instruction-only changes for manual review only', async () => { mockCodingSchemeChangeQueries([10], 12); const responseCountsQb = queryBuilder({ diff --git a/apps/backend/src/app/database/services/coding/coding-freshness.service.ts b/apps/backend/src/app/database/services/coding/coding-freshness.service.ts index f7ce564a..8852f6ca 100644 --- a/apps/backend/src/app/database/services/coding/coding-freshness.service.ts +++ b/apps/backend/src/app/database/services/coding/coding-freshness.service.ts @@ -1236,7 +1236,7 @@ export class CodingFreshnessService { } candidates.add(normalized); - const basename = normalized.split('/').pop(); + const basename = normalized.split(/[\\/]/).filter(Boolean).pop(); if (basename) { candidates.add(basename); } @@ -1256,43 +1256,129 @@ export class CodingFreshnessService { const rows = await this.connection.query( ` - WITH unit_candidates AS ( - SELECT - unit.id AS id, - REGEXP_REPLACE(UPPER(unit.name), '\\.XML$', '', 'i') AS unit_name, - REGEXP_REPLACE(UPPER(COALESCE(unit.alias, '')), '\\.XML$', '', 'i') AS unit_alias, - REGEXP_REPLACE( - UPPER(COALESCE( - (REGEXP_MATCH(unit_file.data, '<\\s*codingschemeref[^>]*>\\s*([^<]+)', 'i'))[1], - '' - )), - '\\.VOCS$', - '', - 'i' - ) AS scheme_ref - FROM "unit" unit - INNER JOIN booklet booklet ON booklet.id = unit.bookletid + WITH scheme_ref_candidates AS ( + SELECT unnest($2::text[]) AS scheme_ref + ), + matching_unit_files AS ( + SELECT DISTINCT unit_file.file_id_normalized AS unit_ref + FROM file_upload unit_file + INNER JOIN scheme_ref_candidates candidate + ON unit_file.coding_scheme_ref_normalized = candidate.scheme_ref + WHERE unit_file.workspace_id = $1 + AND unit_file.file_type = 'Unit' + AND unit_file.file_id_normalized IS NOT NULL + ), + unit_refs AS ( + SELECT scheme_ref AS unit_ref + FROM scheme_ref_candidates + UNION + SELECT unit_ref + FROM matching_unit_files + ), + matched_unit_ids AS ( + SELECT matched_unit.id + FROM unit_refs + CROSS JOIN LATERAL ( + SELECT unit.id, unit.bookletid + FROM "unit" unit + WHERE REGEXP_REPLACE(UPPER(unit.name), '\\.XML$', '', 'i') = unit_refs.unit_ref + UNION + SELECT unit.id, unit.bookletid + FROM "unit" unit + WHERE REGEXP_REPLACE(UPPER(COALESCE(unit.alias, '')), '\\.XML$', '', 'i') = unit_refs.unit_ref + ) matched_unit + INNER JOIN booklet booklet ON booklet.id = matched_unit.bookletid INNER JOIN persons person ON person.id = booklet.personid - LEFT JOIN file_upload unit_file - ON unit_file.workspace_id = person.workspace_id + WHERE person.workspace_id = $1 + ) + SELECT DISTINCT id FROM matched_unit_ids + `, + [workspaceId, schemeRefCandidates] + ) as Array<{ id: number | string }>; + + const indexedUnitIds = this.uniquePositiveIds(rows.map(row => Number(row.id))); + if (!await this.hasLegacyUnitCodingSchemeRefs(workspaceId)) { + return indexedUnitIds; + } + + const legacyRows = await this.connection.query( + ` + WITH legacy_matching_unit_files AS ( + SELECT DISTINCT REGEXP_REPLACE(UPPER(unit_file.file_id), '\\.XML$', '', 'i') AS unit_ref + FROM file_upload unit_file + WHERE unit_file.workspace_id = $1 AND unit_file.file_type = 'Unit' - AND REGEXP_REPLACE(UPPER(unit_file.file_id), '\\.XML$', '', 'i') IN ( - REGEXP_REPLACE(UPPER(unit.name), '\\.XML$', '', 'i'), - REGEXP_REPLACE(UPPER(COALESCE(unit.alias, '')), '\\.XML$', '', 'i') - ) + AND unit_file.file_id_normalized IS NOT NULL + AND unit_file.coding_scheme_ref_normalized IS NULL + AND COALESCE( + REGEXP_REPLACE( + REGEXP_REPLACE( + REGEXP_REPLACE( + UPPER(COALESCE( + NULLIF(unit_file.structured_data #>> '{extractedInfo,codingSchemeRef}', ''), + (REGEXP_MATCH(unit_file.data, '<\\s*codingschemeref[^>]*>\\s*([^<]+)', 'i'))[1], + '' + )), + '\\.VOCS$', + '', + 'i' + ), + '\\.XML$', + '', + 'i' + ), + '^.*[/\\\\]', + '', + 'i' + ), + '' + ) = ANY($2::text[]) + ), + matched_unit_ids AS ( + SELECT matched_unit.id + FROM legacy_matching_unit_files + CROSS JOIN LATERAL ( + SELECT unit.id, unit.bookletid + FROM "unit" unit + WHERE REGEXP_REPLACE(UPPER(unit.name), '\\.XML$', '', 'i') = + legacy_matching_unit_files.unit_ref + UNION + SELECT unit.id, unit.bookletid + FROM "unit" unit + WHERE REGEXP_REPLACE(UPPER(COALESCE(unit.alias, '')), '\\.XML$', '', 'i') = + legacy_matching_unit_files.unit_ref + ) matched_unit + INNER JOIN booklet booklet ON booklet.id = matched_unit.bookletid + INNER JOIN persons person ON person.id = booklet.personid WHERE person.workspace_id = $1 ) - SELECT DISTINCT id - FROM unit_candidates - WHERE unit_name = ANY($2::text[]) - OR unit_alias = ANY($2::text[]) - OR scheme_ref = ANY($2::text[]) - OR REGEXP_REPLACE(scheme_ref, '^.*/', '') = ANY($2::text[]) + SELECT DISTINCT id FROM matched_unit_ids `, [workspaceId, schemeRefCandidates] ) as Array<{ id: number | string }>; - return this.uniquePositiveIds(rows.map(row => Number(row.id))); + return this.uniquePositiveIds([ + ...indexedUnitIds, + ...legacyRows.map(row => Number(row.id)) + ]); + } + + private async hasLegacyUnitCodingSchemeRefs(workspaceId: number): Promise { + const rows = await this.connection.query( + ` + SELECT EXISTS ( + SELECT 1 + FROM file_upload unit_file + WHERE unit_file.workspace_id = $1 + AND unit_file.file_type = 'Unit' + AND unit_file.coding_scheme_ref_normalized IS NULL + LIMIT 1 + ) AS "hasLegacy" + `, + [workspaceId] + ) as Array<{ hasLegacy: boolean | string }>; + + return rows[0]?.hasLegacy === true || rows[0]?.hasLegacy === 'true'; } private async markCodingJobsStaleForAddedUnitIds( diff --git a/apps/backend/src/app/database/services/jobs/job-definition.service.spec.ts b/apps/backend/src/app/database/services/jobs/job-definition.service.spec.ts index e44546b6..fd0674da 100644 --- a/apps/backend/src/app/database/services/jobs/job-definition.service.spec.ts +++ b/apps/backend/src/app/database/services/jobs/job-definition.service.spec.ts @@ -1409,7 +1409,34 @@ describe('JobDefinitionService', () => { ]); }); - it('attaches planned variable usage for listed definitions without created jobs', async () => { + it('does not calculate planned variable usage for listed definitions by default', async () => { + jobDefinitionRepository.find.mockResolvedValue([ + { + id: 4, + workspace_id: 7, + status: 'draft', + assigned_variables: [{ unitName: 'Unit 1', variableId: 'Var 1' }], + assigned_variable_bundles: [], + max_coding_cases: 2, + case_ordering_mode: 'continuous', + distribution_seed: 'seed-4' + } + ]); + + const result = await service.getJobDefinitions(7); + + expect(result[0]).toMatchObject({ + id: 4, + plannedVariableUsage: {}, + planned_variable_usage: {}, + plannedVariableUsageByStatus: {}, + planned_variable_usage_by_status: {} + }); + expect(codingJobService.calculateDistributionVariableUsageByStatusBatch) + .not.toHaveBeenCalled(); + }); + + it('attaches planned variable usage for listed definitions without created jobs when requested', async () => { jobDefinitionRepository.find.mockResolvedValue([ { id: 4, @@ -1428,7 +1455,7 @@ describe('JobDefinitionService', () => { [4, new Map([['Unit 1::Var 1', { regular: 2, deriveError: 0, total: 2 }]])] ])); - const result = await service.getJobDefinitions(7); + const result = await service.getJobDefinitions(7, { includePlannedUsage: true }); expect(result[0]).toMatchObject({ id: 4, @@ -1485,7 +1512,7 @@ describe('JobDefinitionService', () => { ])] ])); - await service.getJobDefinitions(7); + await service.getJobDefinitions(7, { includePlannedUsage: true }); expect(codingJobService.calculateDistributionVariableUsageByStatusBatch).toHaveBeenCalledWith(7, [ expect.objectContaining({ @@ -1533,7 +1560,7 @@ describe('JobDefinitionService', () => { [5, new Map([['Unit 2::Var 2', { regular: 3, deriveError: 0, total: 3 }]])] ])); - const result = await service.getJobDefinitions(7); + const result = await service.getJobDefinitions(7, { includePlannedUsage: true }); expect(codingJobService.calculateDistributionVariableUsageByStatusBatch).toHaveBeenCalledTimes(1); expect(codingJobService.calculateDistributionVariableUsageByStatusBatch).toHaveBeenCalledWith(7, [ diff --git a/apps/backend/src/app/database/services/jobs/job-definition.service.ts b/apps/backend/src/app/database/services/jobs/job-definition.service.ts index 964977f7..c61fff5e 100644 --- a/apps/backend/src/app/database/services/jobs/job-definition.service.ts +++ b/apps/backend/src/app/database/services/jobs/job-definition.service.ts @@ -62,6 +62,10 @@ interface JobDefinitionForUsage { distribution_seed?: string; } +type GetJobDefinitionsOptions = { + includePlannedUsage?: boolean; +}; + export type JobDefinitionWithCreatedJobsCount = JobDefinition & { createdJobsCount: number; created_jobs_count: number; @@ -1158,7 +1162,8 @@ export class JobDefinitionService { } private async attachCreatedJobsCounts( - definitions: JobDefinition[] + definitions: JobDefinition[], + options: GetJobDefinitionsOptions = {} ): Promise { const definitionsByWorkspaceId = new Map(); @@ -1196,53 +1201,55 @@ export class JobDefinitionService { const plannedUsageByStatusByDefinitionId = new Map>(); - const usageRequestPromises: Promise[] = - workspaceDefinitions.map(async definition => { - if (definition.id === undefined) { - return undefined; - } - - const createdJobsCount = countsByDefinitionId.get(definition.id) || 0; - if (createdJobsCount > 0) { - plannedUsageByDefinitionId.set(definition.id, {}); - plannedUsageByStatusByDefinitionId.set(definition.id, {}); - return undefined; - } - - const hydratedBundles = await this.hydrateVariableBundles( - definition.assigned_variable_bundles || [] - ); + if (options.includePlannedUsage) { + const usageRequestPromises: Promise[] = + workspaceDefinitions.map(async definition => { + if (definition.id === undefined) { + return undefined; + } + + const createdJobsCount = countsByDefinitionId.get(definition.id) || 0; + if (createdJobsCount > 0) { + plannedUsageByDefinitionId.set(definition.id, {}); + plannedUsageByStatusByDefinitionId.set(definition.id, {}); + return undefined; + } + + const hydratedBundles = await this.hydrateVariableBundles( + definition.assigned_variable_bundles || [] + ); - return this.buildPlannedVariableUsageBatchRequest(definition.id, { - id: definition.id, - assigned_variables: definition.assigned_variables || [], - assigned_variable_bundles: hydratedBundles, - max_coding_cases: definition.max_coding_cases, - case_ordering_mode: definition.case_ordering_mode, - distribution_seed: this.getDefinitionDistributionSeed(definition) + return this.buildPlannedVariableUsageBatchRequest(definition.id, { + id: definition.id, + assigned_variables: definition.assigned_variables || [], + assigned_variable_bundles: hydratedBundles, + max_coding_cases: definition.max_coding_cases, + case_ordering_mode: definition.case_ordering_mode, + distribution_seed: this.getDefinitionDistributionSeed(definition) + }); }); - }); - const usageRequests = (await Promise.all(usageRequestPromises)) - .filter((request): request is PlannedVariableUsageBatchRequest => request !== undefined); + const usageRequests = (await Promise.all(usageRequestPromises)) + .filter((request): request is PlannedVariableUsageBatchRequest => request !== undefined); - if (usageRequests.length > 0) { - const usageByDefinitionId = await this.codingJobService.calculateDistributionVariableUsageByStatusBatch( - definitionWorkspaceId, - usageRequests - ); + if (usageRequests.length > 0) { + const usageByDefinitionId = await this.codingJobService.calculateDistributionVariableUsageByStatusBatch( + definitionWorkspaceId, + usageRequests + ); - usageRequests.forEach(request => { - if (typeof request.key === 'number') { - const usageByStatus = this.mapVariableUsageByStatusToRecord( - usageByDefinitionId.get(request.key) || new Map() - ); - plannedUsageByStatusByDefinitionId.set(request.key, usageByStatus); - plannedUsageByDefinitionId.set( - request.key, - this.mapVariableUsageByStatusRecordToTotals(usageByStatus) - ); - } - }); + usageRequests.forEach(request => { + if (typeof request.key === 'number') { + const usageByStatus = this.mapVariableUsageByStatusToRecord( + usageByDefinitionId.get(request.key) || new Map() + ); + plannedUsageByStatusByDefinitionId.set(request.key, usageByStatus); + plannedUsageByDefinitionId.set( + request.key, + this.mapVariableUsageByStatusRecordToTotals(usageByStatus) + ); + } + }); + } } workspaceDefinitions.forEach(definition => { @@ -1296,7 +1303,10 @@ export class JobDefinitionService { return definitions as JobDefinitionWithCreatedJobsCount[]; } - async getJobDefinitions(workspaceId?: number): Promise { + async getJobDefinitions( + workspaceId?: number, + options: GetJobDefinitionsOptions = {} + ): Promise { const whereClause = workspaceId ? { workspace_id: workspaceId } : {}; @@ -1310,7 +1320,7 @@ export class JobDefinitionService { await this.hydrateAssignedVariableBundles(definition); } - return this.attachCreatedJobsCounts(definitions); + return this.attachCreatedJobsCounts(definitions, options); } private async assertJobDefinitionHasNoBlockingCreatedJobs(jobDefinition: JobDefinition): Promise { diff --git a/apps/backend/src/app/database/services/workspace/workspace-file-parsing.service.ts b/apps/backend/src/app/database/services/workspace/workspace-file-parsing.service.ts index faad28da..a3e2efc0 100644 --- a/apps/backend/src/app/database/services/workspace/workspace-file-parsing.service.ts +++ b/apps/backend/src/app/database/services/workspace/workspace-file-parsing.service.ts @@ -72,6 +72,20 @@ export class WorkspaceFileParsingService { result.variables = variables; } + const codingSchemeRefs = xmlDocument('CodingSchemeRef, codingSchemeRef') + .map((_, element) => xmlDocument(element).text().trim()) + .get() + .filter(Boolean); + if (codingSchemeRefs.length > 0) { + const normalizedRefs = codingSchemeRefs.map(ref => ref + .toUpperCase() + .replace(/\.VOCS$/i, '') + .replace(/\.XML$/i, '')); + result.codingSchemeRef = codingSchemeRefs[0].toUpperCase(); + result.codingSchemeRefNormalized = normalizedRefs[0]; + result.codingSchemeRefs = Array.from(new Set(normalizedRefs)); + } + const definitions = xmlDocument('Definition'); if (definitions.length) { const definitionsArray: Array> = []; diff --git a/apps/backend/src/app/database/services/workspace/workspace-files.service.spec.ts b/apps/backend/src/app/database/services/workspace/workspace-files.service.spec.ts index 2fce7c97..ef5f3ce4 100644 --- a/apps/backend/src/app/database/services/workspace/workspace-files.service.spec.ts +++ b/apps/backend/src/app/database/services/workspace/workspace-files.service.spec.ts @@ -2,6 +2,7 @@ import { ConsoleLogger, Logger } from '@nestjs/common'; import { WorkspaceFilesService } from './workspace-files.service'; import { FileIo } from '../../../admin/workspace/file-io.interface'; import { getManualCodingScopeKey } from '../../utils/manual-coding-scope.util'; +import { NO_CODING_SCHEME_REF_NORMALIZED } from '../../entities/file_upload.entity'; describe('WorkspaceFilesService.handleFile', () => { beforeAll(() => { @@ -244,6 +245,13 @@ describe('WorkspaceFilesService coding scheme freshness', () => { const mockCodingReadinessCacheInvalidator = { invalidateWorkspaceReadinessCache: jest.fn() }; + const mockWorkspaceFileParsingService = { + extractUnitInfo: jest.fn().mockResolvedValue({ + codingSchemeRef: 'UNIT_A.VOCS', + codingSchemeRefNormalized: 'UNIT_A', + codingSchemeRefs: ['UNIT_A'] + }) + }; function makeService(): WorkspaceFilesService { return new WorkspaceFilesService( @@ -254,7 +262,7 @@ describe('WorkspaceFilesService coding scheme freshness', () => { mockCodingStatisticsService as unknown as CtorParams[4], {} as unknown as CtorParams[5], {} as unknown as CtorParams[6], - {} as unknown as CtorParams[7], + mockWorkspaceFileParsingService as unknown as CtorParams[7], {} as unknown as CtorParams[8], {} as unknown as CtorParams[9], { delete: jest.fn() } as unknown as CtorParams[10], @@ -269,6 +277,11 @@ describe('WorkspaceFilesService coding scheme freshness', () => { beforeEach(() => { jest.clearAllMocks(); + mockWorkspaceFileParsingService.extractUnitInfo.mockResolvedValue({ + codingSchemeRef: 'UNIT_A.VOCS', + codingSchemeRefNormalized: 'UNIT_A', + codingSchemeRefs: ['UNIT_A'] + }); jest.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined); jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined); jest.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); @@ -414,6 +427,106 @@ describe('WorkspaceFilesService coding scheme freshness', () => { expect(mockFileUploadRepository.upsert).toHaveBeenCalled(); }); + it('keeps incomplete variable cache fresh after a pure coding scheme upload', async () => { + const service = makeService(); + const oldData = createCodingScheme({ processing: [] }); + const newData = createCodingScheme({ processing: ['IGNORE_CASE'] }); + mockFileUploadRepository.findOne.mockResolvedValue({ + file_id: 'UNIT_A.VOCS', + data: oldData + }); + + await service.uploadTestFiles(1, [createVocsFile(newData)], true); + + expect(mockCodingStatisticsService.invalidateCache).not.toHaveBeenCalled(); + expect(mockCodingStatisticsService.invalidateIncompleteVariablesCache) + .toHaveBeenCalledWith(1); + expect(mockCodingReadinessCacheInvalidator.invalidateWorkspaceReadinessCache) + .toHaveBeenCalledWith(1); + }); + + it('extracts coding scheme refs for unit XML files uploaded as octet-stream', async () => { + const service = makeService(); + const unitXml = ` + + UNIT_A + UNIT_A.VOCS + + `; + mockFileUploadRepository.findOne.mockResolvedValue(null); + + await ( + service as unknown as { + handleOctetStreamFile: (...args: unknown[]) => Promise; + } + ).handleOctetStreamFile( + 1, + { + fieldname: 'files', + originalname: 'unit_a.xml', + encoding: '7bit', + mimetype: 'application/octet-stream', + buffer: Buffer.from(unitXml), + size: Buffer.byteLength(unitXml) + }, + true + ); + + expect(mockWorkspaceFileParsingService.extractUnitInfo).toHaveBeenCalled(); + expect(mockFileUploadRepository.upsert).toHaveBeenCalledWith( + expect.objectContaining({ + file_id_normalized: 'UNIT_A', + coding_scheme_ref_normalized: 'UNIT_A', + file_type: 'Unit', + structured_data: { + extractedInfo: expect.objectContaining({ + rootElement: 'Unit', + detectedVia: 'octet-stream-handler', + codingSchemeRefNormalized: 'UNIT_A' + }) + } + }), + ['file_id', 'workspace_id'] + ); + }); + + it('marks Unit files without coding scheme refs as normalized no-ref lookups', async () => { + const service = makeService(); + const unitXml = ` + + UNIT_WITHOUT_SCHEME + + `; + mockFileUploadRepository.findOne.mockResolvedValue(null); + mockWorkspaceFileParsingService.extractUnitInfo.mockResolvedValue({}); + + await ( + service as unknown as { + handleOctetStreamFile: (...args: unknown[]) => Promise; + } + ).handleOctetStreamFile( + 1, + { + fieldname: 'files', + originalname: 'unit_without_scheme.xml', + encoding: '7bit', + mimetype: 'application/octet-stream', + buffer: Buffer.from(unitXml), + size: Buffer.byteLength(unitXml) + }, + true + ); + + expect(mockFileUploadRepository.upsert).toHaveBeenCalledWith( + expect.objectContaining({ + file_id_normalized: 'UNIT_WITHOUT_SCHEME', + coding_scheme_ref_normalized: NO_CODING_SCHEME_REF_NORMALIZED, + file_type: 'Unit' + }), + ['file_id', 'workspace_id'] + ); + }); + it('invalidates workspace file caches after Testcenter import writes files', async () => { const service = makeService(); const conflictQueryBuilder = { @@ -456,6 +569,51 @@ describe('WorkspaceFilesService coding scheme freshness', () => { expect(invalidateSpy).toHaveBeenCalledWith(1); }); + it('extracts coding scheme refs for Unit files imported from Testcenter', async () => { + const service = makeService(); + const conflictQueryBuilder = { + select: jest.fn().mockReturnThis(), + where: jest.fn().mockReturnThis(), + andWhere: jest.fn().mockReturnThis(), + getMany: jest.fn().mockResolvedValue([]) + }; + const insertQueryBuilder = { + insert: jest.fn().mockReturnThis(), + into: jest.fn().mockReturnThis(), + values: jest.fn().mockReturnThis(), + orIgnore: jest.fn().mockReturnThis(), + execute: jest.fn().mockResolvedValue(undefined) + }; + mockFileUploadRepository.createQueryBuilder + .mockReturnValueOnce(conflictQueryBuilder) + .mockReturnValueOnce(insertQueryBuilder); + + await service.testCenterImport([ + { + workspace_id: 1, + file_id: 'UnitA', + filename: 'UnitA.xml', + file_type: 'Unit', + file_size: 12, + data: 'UnitA.vocs' + } + ]); + + expect(mockWorkspaceFileParsingService.extractUnitInfo).toHaveBeenCalled(); + expect(insertQueryBuilder.values).toHaveBeenCalledWith([ + expect.objectContaining({ + file_id: 'UnitA', + file_id_normalized: 'UNITA', + coding_scheme_ref_normalized: 'UNIT_A', + structured_data: { + extractedInfo: expect.objectContaining({ + codingSchemeRefNormalized: 'UNIT_A' + }) + } + }) + ]); + }); + it('invalidates auto-coding readiness cache when workspace file caches are invalidated', async () => { const service = makeService(); diff --git a/apps/backend/src/app/database/services/workspace/workspace-files.service.ts b/apps/backend/src/app/database/services/workspace/workspace-files.service.ts index 7233eb43..5ca0b9c3 100644 --- a/apps/backend/src/app/database/services/workspace/workspace-files.service.ts +++ b/apps/backend/src/app/database/services/workspace/workspace-files.service.ts @@ -16,6 +16,7 @@ import * as path from 'path'; import { parseStringPromise } from 'xml2js'; import { VariableInfo } from '@iqbspecs/variable-info/variable-info.interface'; import FileUpload, { + NO_CODING_SCHEME_REF_NORMALIZED, StructuredFileData } from '../../entities/file_upload.entity'; import { FilesDto } from '../../../../../../../api-dto/files/files.dto'; @@ -136,6 +137,12 @@ interface CodingSchemeFreshnessImpact { manualCodingChanged: boolean; } +type UploadedFileForCacheInvalidation = { + fileId?: string; + filename?: string; + fileType?: string; +}; + @Injectable() export class WorkspaceFilesService implements OnModuleInit { private readonly logger = new Logger(WorkspaceFilesService.name); @@ -287,6 +294,94 @@ export class WorkspaceFilesService implements OnModuleInit { return String(fileId || '').trim().toUpperCase().endsWith('.VOCS'); } + private isUploadedCodingSchemeFile( + file: UploadedFileForCacheInvalidation + ): boolean { + return [file.fileId, file.filename] + .some(value => this.isCodingSchemeFileId(value)); + } + + private shouldInvalidateCodingStatisticsAfterUpload( + result: TestFilesUploadResultDto + ): boolean { + const uploadedFiles = result.uploadedFiles || []; + if (uploadedFiles.length === 0) { + return false; + } + return !uploadedFiles.every(file => this.isUploadedCodingSchemeFile(file)); + } + + private hasUploadedFiles(result: TestFilesUploadResultDto): boolean { + return (result.uploadedFiles || []).length > 0; + } + + private mergeExtractedFileInfo( + existingStructuredData: unknown, + extractedInfo: Record + ): StructuredFileData { + const existing = + existingStructuredData && + typeof existingStructuredData === 'object' && + !Array.isArray(existingStructuredData) ? + existingStructuredData as StructuredFileData : + {}; + + return { + ...existing, + extractedInfo: { + ...(existing.extractedInfo || {}), + ...extractedInfo + } + }; + } + + private async enrichTestCenterUnitEntry( + entry: Record + ): Promise> { + if (String(entry.file_type || '').toLowerCase() !== 'unit') { + return entry; + } + if (entry.data === null || entry.data === undefined) { + return entry; + } + + try { + const xmlContent = Buffer.isBuffer(entry.data) ? + entry.data.toString('utf8') : + String(entry.data); + const xmlDocument = cheerio.load(xmlContent, { xmlMode: true }); + const extractedInfo = + await this.workspaceFileParsingService.extractUnitInfo(xmlDocument); + + if (Object.keys(extractedInfo).length === 0) { + return entry; + } + + return { + ...entry, + structured_data: this.mergeExtractedFileInfo( + entry.structured_data, + extractedInfo + ) + }; + } catch (error) { + this.logger.warn( + `Could not extract Unit metadata from Testcenter import file ${ + String(entry.file_id || entry.filename || '') + }: ${error instanceof Error ? error.message : 'Unknown error'}` + ); + return entry; + } + } + + private async enrichTestCenterImportEntries( + entries: Record[] + ): Promise[]> { + const enrichedEntries = + await Promise.all(entries.map(entry => this.enrichTestCenterUnitEntry(entry))); + return enrichedEntries.map(entry => this.withNormalizedFileLookupFields(entry)); + } + private normalizeCodingSchemeUnitName(fileId: unknown): string { return String(fileId || '') .trim() @@ -294,6 +389,61 @@ export class WorkspaceFilesService implements OnModuleInit { .replace(/\.VOCS$/i, ''); } + private normalizeFileIdForLookup(fileId: unknown): string | null { + const normalized = String(fileId || '') + .trim() + .toUpperCase() + .replace(/\.XML$/i, ''); + return normalized || null; + } + + private normalizeCodingSchemeRefForLookup(value: unknown): string | null { + const normalized = String(value || '') + .trim() + .toUpperCase() + .replace(/\.VOCS$/i, '') + .replace(/\.XML$/i, '') + .split(/[\\/]/) + .filter(Boolean) + .pop() || ''; + return normalized || null; + } + + private getCodingSchemeRefForLookup( + fileType: unknown, + structuredData: StructuredFileData | null | undefined + ): string | null { + if (String(fileType || '').toLowerCase() !== 'unit') { + return null; + } + + const extractedInfo = structuredData?.extractedInfo || {}; + const refs = extractedInfo.codingSchemeRefs; + if (Array.isArray(refs) && refs.length > 0) { + return this.normalizeCodingSchemeRefForLookup(refs[0]) || + NO_CODING_SCHEME_REF_NORMALIZED; + } + + return this.normalizeCodingSchemeRefForLookup( + extractedInfo.codingSchemeRefNormalized || + extractedInfo.codingSchemeRef + ) || NO_CODING_SCHEME_REF_NORMALIZED; + } + + private withNormalizedFileLookupFields( + entry: Record + ): Record { + const structuredData = entry.structured_data as StructuredFileData | null | undefined; + return { + ...entry, + file_id_normalized: this.normalizeFileIdForLookup(entry.file_id), + coding_scheme_ref_normalized: this.getCodingSchemeRefForLookup( + entry.file_type, + structuredData + ) + }; + } + private parseCodingSchemeData(data: unknown): ParsedCodingScheme | null { if (data === null || data === undefined) { return null; @@ -1162,10 +1312,14 @@ ${bookletRefs} // Invalidate memory caches inside this service await this.invalidateWorkspaceFileCaches(workspace_id); - await this.codingStatisticsService.invalidateCache(workspace_id); - await this.codingStatisticsService.invalidateIncompleteVariablesCache( - workspace_id - ); + if (this.shouldInvalidateCodingStatisticsAfterUpload(result)) { + await this.codingStatisticsService.invalidateCache(workspace_id); + } + if (this.hasUploadedFiles(result)) { + await this.codingStatisticsService.invalidateIncompleteVariablesCache( + workspace_id + ); + } return result; } catch (error) { this.logger.error( @@ -1505,7 +1659,7 @@ ${bookletRefs} }; await this.fileUploadRepository.upsert( - { + this.withNormalizedFileLookupFields({ workspace_id: workspaceId, filename: file.originalname, file_type: fileType, @@ -1514,7 +1668,7 @@ ${bookletRefs} data: file.buffer.toString(), file_id: resolvedFileId, structured_data: structuredData - }, + }), ['file_id', 'workspace_id'] ); @@ -1577,7 +1731,7 @@ ${bookletRefs} }; } await this.fileUploadRepository.upsert( - { + this.withNormalizedFileLookupFields({ filename: file.originalname, workspace_id: workspaceId, file_type: 'Schemer', @@ -1586,7 +1740,7 @@ ${bookletRefs} file_id: resourceFileId, data: file.buffer.toString(), structured_data: structuredData - }, + }), ['file_id', 'workspace_id'] ); @@ -1618,7 +1772,7 @@ ${bookletRefs} }; } await this.fileUploadRepository.upsert( - { + this.withNormalizedFileLookupFields({ filename: file.originalname, workspace_id: workspaceId, file_type: 'Resource', @@ -1627,7 +1781,7 @@ ${bookletRefs} file_id: resourceFileId, data: file.buffer.toString(), structured_data: structuredData - }, + }), ['file_id', 'workspace_id'] ); @@ -1640,7 +1794,7 @@ ${bookletRefs} const resourceFileId = this.workspaceFileParsingService.getResourceId(file); await this.fileUploadRepository.upsert( - { + this.withNormalizedFileLookupFields({ filename: file.originalname, workspace_id: workspaceId, file_type: 'Resource', @@ -1649,7 +1803,7 @@ ${bookletRefs} file_id: resourceFileId, data: file.buffer.toString(), structured_data: { metadata: {} } - }, + }), ['file_id', 'workspace_id'] ); @@ -1674,7 +1828,7 @@ ${bookletRefs} const fileExtension = path.extname(file.originalname).toLowerCase(); let fileType = 'Resource'; let fileContent: string | Buffer; - let extractedInfo = {}; + let extractedInfo: Record = {}; const textFileExtensions = [ '.xml', @@ -1716,7 +1870,8 @@ ${bookletRefs} fileType = 'Unit'; extractedInfo = { rootElement: 'Unit', - detectedVia: 'octet-stream-handler' + detectedVia: 'octet-stream-handler', + ...await this.workspaceFileParsingService.extractUnitInfo($) }; } else if ($('SysCheck').length > 0) { fileType = 'SysCheck'; @@ -1736,16 +1891,18 @@ ${bookletRefs} extractedInfo }; - const fileUpload = this.fileUploadRepository.create({ - workspace_id: workspaceId, - filename: file.originalname, - file_id: file.originalname.toUpperCase(), - file_type: fileType, - file_size: file.size, - created_at: new Date() as unknown as number, - data: fileContent, - structured_data: structuredData - }); + const fileUpload = this.fileUploadRepository.create( + this.withNormalizedFileLookupFields({ + workspace_id: workspaceId, + filename: file.originalname, + file_id: file.originalname.toUpperCase(), + file_type: fileType, + file_size: file.size, + created_at: new Date() as unknown as number, + data: fileContent, + structured_data: structuredData + }) + ); const existing = await this.fileUploadRepository.findOne({ where: { file_id: fileUpload.file_id, workspace_id: workspaceId } @@ -1957,7 +2114,9 @@ ${bookletRefs} overwriteFileIds?: string[] ): Promise { try { - const normalized = Array.isArray(entries) ? entries : []; + const normalized = await this.enrichTestCenterImportEntries( + Array.isArray(entries) ? entries : [] + ); const workspaceId = Number( (normalized[0] as { workspace_id?: unknown } | undefined)?.workspace_id ); diff --git a/apps/frontend/src/app/coding/components/coding-job-definition-dialog/coding-job-definition-dialog.component.ts b/apps/frontend/src/app/coding/components/coding-job-definition-dialog/coding-job-definition-dialog.component.ts index 4dfa49e8..11e5425f 100644 --- a/apps/frontend/src/app/coding/components/coding-job-definition-dialog/coding-job-definition-dialog.component.ts +++ b/apps/frontend/src/app/coding/components/coding-job-definition-dialog/coding-job-definition-dialog.component.ts @@ -534,7 +534,10 @@ export class CodingJobDefinitionDialogComponent implements OnInit, OnDestroy { return; } - this.codingJobBackendService.getJobDefinitions(workspaceId).subscribe({ + this.codingJobBackendService.getJobDefinitions( + workspaceId, + { includePlannedUsage: true } + ).subscribe({ next: definitions => { // When editing an existing job definition, exclude the current job definition // from the list to prevent its variables from being incorrectly disabled diff --git a/apps/frontend/src/app/coding/components/scheme-editor-dialog/scheme-editor-dialog.component.spec.ts b/apps/frontend/src/app/coding/components/scheme-editor-dialog/scheme-editor-dialog.component.spec.ts index 73ddeaca..39eee510 100644 --- a/apps/frontend/src/app/coding/components/scheme-editor-dialog/scheme-editor-dialog.component.spec.ts +++ b/apps/frontend/src/app/coding/components/scheme-editor-dialog/scheme-editor-dialog.component.spec.ts @@ -166,8 +166,20 @@ describe('SchemeEditorDialogComponent', () => { true, ['test-scheme.json'] ); - expect(mockSnackBar.open).toHaveBeenCalledWith('coding.schemer.save-success', 'Success', expect.any(Object)); + expect(mockSnackBar.open).toHaveBeenCalledWith( + 'coding.schemer.save-success', + 'coding.schemer.check-coding-status', + { duration: 10000 } + ); expect(mockDialogRef.close).toHaveBeenCalledWith(true); + + snackBarAction$.next(); + tick(); + + expect(mockRouter.navigate).toHaveBeenCalledWith( + ['/workspace-admin/1/coding/management'], + { queryParams: { refreshCodingFreshness: '1' } } + ); })); it('should show a freshness warning with navigation action after saving', fakeAsync(() => { @@ -230,7 +242,11 @@ describe('SchemeEditorDialogComponent', () => { expect(mockFileService.deleteFiles).not.toHaveBeenCalled(); expect(mockFileService.uploadTestFiles).toHaveBeenCalled(); - expect(mockSnackBar.open).toHaveBeenCalledWith('coding.schemer.save-success', 'Success', expect.any(Object)); + expect(mockSnackBar.open).toHaveBeenCalledWith( + 'coding.schemer.save-success', + 'coding.schemer.check-coding-status', + { duration: 10000 } + ); expect(mockDialogRef.close).toHaveBeenCalledWith(true); })); diff --git a/apps/frontend/src/app/coding/components/scheme-editor-dialog/scheme-editor-dialog.component.ts b/apps/frontend/src/app/coding/components/scheme-editor-dialog/scheme-editor-dialog.component.ts index 84d92498..2dc59a15 100644 --- a/apps/frontend/src/app/coding/components/scheme-editor-dialog/scheme-editor-dialog.component.ts +++ b/apps/frontend/src/app/coding/components/scheme-editor-dialog/scheme-editor-dialog.component.ts @@ -290,11 +290,7 @@ export class SchemeEditorDialogComponent implements OnInit { if (this.hasCodingFreshnessWarning(result.issues)) { this.showCodingFreshnessWarning(); } else { - this.snackBar.open( - this.translate.instant('coding.schemer.save-success'), - 'Success', - { duration: 3000 } - ); + this.showCodingFreshnessSuccess(); } this.dialogRef.close(true); } else { @@ -320,6 +316,22 @@ export class SchemeEditorDialogComponent implements OnInit { { duration: 10000 } ) as ReturnType | undefined; + this.navigateToCodingStatusOnAction(snackBarRef); + } + + private showCodingFreshnessSuccess(): void { + const snackBarRef = this.snackBar.open( + this.translate.instant('coding.schemer.save-success'), + this.translate.instant('coding.schemer.check-coding-status'), + { duration: 10000 } + ) as ReturnType | undefined; + + this.navigateToCodingStatusOnAction(snackBarRef); + } + + private navigateToCodingStatusOnAction( + snackBarRef: ReturnType | undefined + ): void { snackBarRef?.onAction().subscribe(() => { this.router.navigate( [`/workspace-admin/${this.data.workspaceId}/coding/management`], diff --git a/apps/frontend/src/app/coding/services/coding-job-backend.service.spec.ts b/apps/frontend/src/app/coding/services/coding-job-backend.service.spec.ts index c0e40568..f47f9417 100644 --- a/apps/frontend/src/app/coding/services/coding-job-backend.service.spec.ts +++ b/apps/frontend/src/app/coding/services/coding-job-backend.service.spec.ts @@ -266,6 +266,16 @@ describe('CodingJobBackendService', () => { ]); }); + it('should request planned usage when asked', () => { + service.getJobDefinitions(1, { includePlannedUsage: true }).subscribe(); + + const req = httpMock.expectOne( + `${mockServerUrl}admin/workspace/1/coding/job-definitions?includePlannedUsage=true` + ); + expect(req.request.method).toBe('GET'); + req.flush([]); + }); + it('should create coding jobs through the dedicated job definition endpoint', () => { service.createCodingJobFromDefinition(1, 42).subscribe(response => { expect(response.success).toBe(true); diff --git a/apps/frontend/src/app/coding/services/coding-job-backend.service.ts b/apps/frontend/src/app/coding/services/coding-job-backend.service.ts index 695a7e6d..6f5df35c 100644 --- a/apps/frontend/src/app/coding/services/coding-job-backend.service.ts +++ b/apps/frontend/src/app/coding/services/coding-job-backend.service.ts @@ -949,10 +949,16 @@ export class CodingJobBackendService { ); } - getJobDefinitions(workspaceId: number): Observable { + getJobDefinitions( + workspaceId: number, + options: { includePlannedUsage?: boolean } = {} + ): Observable { const url = `${this.serverUrl}admin/workspace/${workspaceId}/coding/job-definitions`; + const params = options.includePlannedUsage ? + new HttpParams().set('includePlannedUsage', 'true') : + undefined; return this.http - .get(url, { headers: this.authHeader }) + .get(url, { headers: this.authHeader, params }) .pipe( map((definitions: JobDefinitionApiResponse[]) => definitions.map(def => ({ id: def.id, diff --git a/apps/frontend/src/app/ws-admin/components/test-files/test-files-upload-result-dialog.component.spec.ts b/apps/frontend/src/app/ws-admin/components/test-files/test-files-upload-result-dialog.component.spec.ts index c8e0b75f..ff7661ef 100644 --- a/apps/frontend/src/app/ws-admin/components/test-files/test-files-upload-result-dialog.component.spec.ts +++ b/apps/frontend/src/app/ws-admin/components/test-files/test-files-upload-result-dialog.component.spec.ts @@ -97,6 +97,7 @@ describe('TestFilesUploadResultDialogComponent', () => { component.filterText = 'nomatch'; expect(component.filteredUploadedFiles).toHaveLength(0); expect(component.hasCodingFreshnessWarning).toBe(true); + expect(component.hasUploadedCodingScheme).toBe(false); expect(component.canCheckCodingStatus).toBe(true); expect(component.trackByUploaded(0, data.uploadedFiles[0])).toContain('booklet.xml'); @@ -142,6 +143,34 @@ describe('TestFilesUploadResultDialogComponent', () => { ); }); + it('allows checking coding status after successful coding scheme uploads without freshness warnings', async () => { + TestBed.resetTestingModule(); + await TestBed.configureTestingModule({ + imports: [TestFilesUploadResultDialogComponent, NoopAnimationsModule], + providers: [ + { provide: MatDialogRef, useValue: dialogRef }, + { provide: Router, useValue: router }, + { + provide: MAT_DIALOG_DATA, + useValue: { + workspaceId: 1, + attempted: 1, + uploadedFiles: [{ filename: 'UNIT_A.vocs', fileId: 'UNIT_A.VOCS', fileType: 'Resource' } as never], + failedFiles: [], + remainingConflicts: [], + issues: [] + } satisfies TestFilesUploadResultDialogData + } + ] + }).compileComponents(); + + component = TestBed.createComponent(TestFilesUploadResultDialogComponent).componentInstance; + + expect(component.hasCodingFreshnessWarning).toBe(false); + expect(component.hasUploadedCodingScheme).toBe(true); + expect(component.canCheckCodingStatus).toBe(true); + }); + it('falls back to defaults when optional data is absent', async () => { TestBed.resetTestingModule(); await TestBed.configureTestingModule({ @@ -172,5 +201,6 @@ describe('TestFilesUploadResultDialogComponent', () => { expect(component.remainingConflictsCount).toBe(0); expect(component.overwriteSelectedCount).toBe(0); expect(component.filteredIssues).toEqual([]); + expect(component.canCheckCodingStatus).toBe(false); }); }); diff --git a/apps/frontend/src/app/ws-admin/components/test-files/test-files-upload-result-dialog.component.ts b/apps/frontend/src/app/ws-admin/components/test-files/test-files-upload-result-dialog.component.ts index 85c32062..cc939d05 100644 --- a/apps/frontend/src/app/ws-admin/components/test-files/test-files-upload-result-dialog.component.ts +++ b/apps/frontend/src/app/ws-admin/components/test-files/test-files-upload-result-dialog.component.ts @@ -208,8 +208,18 @@ export class TestFilesUploadResultDialogComponent { return this.issues.some(issue => issue.category === 'coding_freshness'); } + get hasUploadedCodingScheme(): boolean { + return this.uploadedFiles.some(file => this.isCodingSchemeFile(file)); + } + get canCheckCodingStatus(): boolean { - return !!this.data.workspaceId && this.hasCodingFreshnessWarning; + return !!this.data.workspaceId && + (this.hasCodingFreshnessWarning || this.hasUploadedCodingScheme); + } + + private isCodingSchemeFile(file: TestFilesUploadUploadedDto): boolean { + return [file.fileId, file.filename] + .some(value => String(value || '').trim().toUpperCase().endsWith('.VOCS')); } private matchesQuery( diff --git a/database/changelog/coding-box.changelog-1.16.3.sql b/database/changelog/coding-box.changelog-1.16.3.sql new file mode 100644 index 00000000..db0a9191 --- /dev/null +++ b/database/changelog/coding-box.changelog-1.16.3.sql @@ -0,0 +1,171 @@ +-- liquibase formatted sql + +-- changeset julian:1 +-- comment: Backfill normalized coding scheme refs for existing Unit files + +WITH extracted_refs AS ( + SELECT + f."id", + NULLIF( + UPPER(BTRIM((match."ref_match")[1])), + '' + ) AS "coding_scheme_ref" + FROM "public"."file_upload" f + CROSS JOIN LATERAL regexp_matches( + f."data", + '<[[:space:]]*codingschemeref[^>]*>[[:space:]]*([^<]+)', + 'i' + ) AS match("ref_match") + WHERE f."file_type" = 'Unit' + AND COALESCE( + f."structured_data" #>> '{extractedInfo,codingSchemeRefNormalized}', + '' + ) = '' +), +normalized_refs AS ( + SELECT + "id", + "coding_scheme_ref", + NULLIF( + regexp_replace( + regexp_replace("coding_scheme_ref", '\.VOCS$', '', 'i'), + '\.XML$', + '', + 'i' + ), + '' + ) AS "coding_scheme_ref_normalized" + FROM extracted_refs + WHERE "coding_scheme_ref" IS NOT NULL +) +UPDATE "public"."file_upload" f +SET "structured_data" = jsonb_set( + COALESCE(f."structured_data", '{}'::jsonb), + '{extractedInfo}', + COALESCE(f."structured_data" #> '{extractedInfo}', '{}'::jsonb) || + jsonb_build_object( + 'codingSchemeRef', + refs."coding_scheme_ref", + 'codingSchemeRefNormalized', + refs."coding_scheme_ref_normalized", + 'codingSchemeRefs', + jsonb_build_array(refs."coding_scheme_ref_normalized") + ), + true +) +FROM normalized_refs refs +WHERE f."id" = refs."id" + AND refs."coding_scheme_ref_normalized" IS NOT NULL; + +-- rollback -- Cannot safely restore previous structured_data automatically; this backfill only adds missing extracted Unit coding scheme refs. + +-- changeset julian:2 +-- comment: Add indexed normalized file lookup columns for coding freshness + +ALTER TABLE "public"."file_upload" + ADD COLUMN IF NOT EXISTS "file_id_normalized" VARCHAR(100), + ADD COLUMN IF NOT EXISTS "coding_scheme_ref_normalized" VARCHAR(100); + +UPDATE "public"."file_upload" +SET "file_id_normalized" = NULLIF( + regexp_replace( + UPPER(BTRIM(COALESCE("file_id", ''))), + '\.XML$', + '', + 'i' + ), + '' +) +WHERE "file_id_normalized" IS NULL; + +WITH structured_refs AS ( + SELECT + f."id", + NULLIF( + regexp_replace( + regexp_replace( + regexp_replace( + UPPER(BTRIM(COALESCE( + f."structured_data" #>> '{extractedInfo,codingSchemeRefNormalized}', + f."structured_data" #>> '{extractedInfo,codingSchemeRef}', + '' + ))), + '\.VOCS$', + '', + 'i' + ), + '\.XML$', + '', + 'i' + ), + '^.*[/\\]', + '', + 'i' + ), + '' + ) AS "coding_scheme_ref_normalized" + FROM "public"."file_upload" f + WHERE f."file_type" = 'Unit' + AND f."coding_scheme_ref_normalized" IS NULL +) +UPDATE "public"."file_upload" f +SET "coding_scheme_ref_normalized" = refs."coding_scheme_ref_normalized" +FROM structured_refs refs +WHERE f."id" = refs."id" + AND refs."coding_scheme_ref_normalized" IS NOT NULL; + +WITH extracted_refs AS ( + SELECT + f."id", + NULLIF( + regexp_replace( + regexp_replace( + regexp_replace(UPPER(BTRIM((match."ref_match")[1])), '\.VOCS$', '', 'i'), + '\.XML$', + '', + 'i' + ), + '^.*[/\\]', + '', + 'i' + ), + '' + ) AS "coding_scheme_ref_normalized" + FROM "public"."file_upload" f + CROSS JOIN LATERAL regexp_matches( + f."data", + '<[[:space:]]*codingschemeref[^>]*>[[:space:]]*([^<]+)', + 'i' + ) AS match("ref_match") + WHERE f."file_type" = 'Unit' + AND f."coding_scheme_ref_normalized" IS NULL +) +UPDATE "public"."file_upload" f +SET "coding_scheme_ref_normalized" = refs."coding_scheme_ref_normalized" +FROM extracted_refs refs +WHERE f."id" = refs."id" + AND refs."coding_scheme_ref_normalized" IS NOT NULL; + +UPDATE "public"."file_upload" +SET "coding_scheme_ref_normalized" = '__NO_CODING_SCHEME_REF__' +WHERE "file_type" = 'Unit' + AND "coding_scheme_ref_normalized" IS NULL; + +CREATE INDEX IF NOT EXISTS "idx_file_upload_workspace_type_file_id_norm" + ON "public"."file_upload" ("workspace_id", "file_type", "file_id_normalized"); + +CREATE INDEX IF NOT EXISTS "idx_file_upload_workspace_type_scheme_ref_norm" + ON "public"."file_upload" ("workspace_id", "file_type", "coding_scheme_ref_normalized"); + +CREATE INDEX IF NOT EXISTS "idx_unit_name_normalized" + ON "public"."unit" ((regexp_replace(UPPER("name"), '\.XML$', '', 'i'))); + +CREATE INDEX IF NOT EXISTS "idx_unit_alias_normalized" + ON "public"."unit" ((regexp_replace(UPPER(COALESCE("alias", '')), '\.XML$', '', 'i'))); + +-- rollback DROP INDEX IF EXISTS "public"."idx_unit_alias_normalized"; +-- rollback DROP INDEX IF EXISTS "public"."idx_unit_name_normalized"; +-- rollback DROP INDEX IF EXISTS "public"."idx_file_upload_workspace_type_scheme_ref_norm"; +-- rollback DROP INDEX IF EXISTS "public"."idx_file_upload_workspace_type_file_id_norm"; +-- rollback ALTER TABLE "public"."file_upload" DROP COLUMN IF EXISTS "coding_scheme_ref_normalized"; +-- rollback ALTER TABLE "public"."file_upload" DROP COLUMN IF EXISTS "file_id_normalized"; diff --git a/database/changelog/coding-box.changelog-root.xml b/database/changelog/coding-box.changelog-root.xml index 4fa6220f..63bf28f3 100644 --- a/database/changelog/coding-box.changelog-root.xml +++ b/database/changelog/coding-box.changelog-root.xml @@ -41,4 +41,5 @@ +