Skip to content

DMP-5561 ARM - Add further checks on ARM upload status #3291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stephenokeefe merged 6 commits into from
Jul 16, 2026
Merged

DMP-5561 ARM - Add further checks on ARM upload status #3291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
280f6db
DMP-5561 ARM - Add further checks on ARM upload status
karen-hedges Jun 8, 2026
1b3b0d4
DMP-5561 ARM - Add further checks on ARM upload status
karen-hedges Jun 8, 2026
5a8fc5e
DMP-5561 ARM - Add further checks on ARM upload status
karen-hedges Jun 9, 2026
fe3755a
DMP-5561 ARM - Add further checks on ARM upload status
karen-hedges Jun 9, 2026
dbe0fbe
DMP-5561 ARM - Add further checks on ARM upload status
karen-hedges Jun 9, 2026
dad792e
DMP-5561 ARM - Add further checks on ARM upload status
karen-hedges Jun 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions src/integrationTest/java/uk/gov/hmcts/darts/arm/helper/DataStoreToArmHelperIntTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ void ignoreArmDropZoneStatus() {
externalObjectDirectoryStub.createAndSaveEod(medias.get(1), ARM_DROP_ZONE, ARM);

List<Long> eodEntitiesToSendToArm = dataStoreToArmHelper.getEodEntitiesToSendToArm(EodHelper.unstructuredLocation(),
EodHelper.armLocation(), 5);
EodHelper.armLocation(), 5);
assertEquals(1, eodEntitiesToSendToArm.size());

}
Expand All @@ -139,7 +139,7 @@ void getCorrectEodEntities() {
dartsDatabase.save(failedTooManyTimesEod);

List<Long> eodEntitiesToSendToArm = dataStoreToArmHelper.getEodEntitiesToSendToArm(EodHelper.unstructuredLocation(),
EodHelper.armLocation(), 5);
EodHelper.armLocation(), 5);
assertEquals(3, eodEntitiesToSendToArm.size());

}
Expand Down Expand Up @@ -262,8 +262,10 @@ void shouldPushRawDataToArm() {
ArmBatchItem batchItem = new ArmBatchItem();
ObjectRecordStatusEntity status = externalObjectDirectoryStub.getStatus(ARM_INGESTION);
batchItem.setPreviousStatus(status);
String rawFilename = dataStoreToArmHelper.generateRawFilename(externalObjectDirectory);
UserAccountEntity userAccount = dartsDatabase.getUserAccountStub().getIntegrationTestUserAccountEntity();

boolean result = dataStoreToArmHelper.shouldPushRawDataToArm(batchItem);
boolean result = dataStoreToArmHelper.shouldPushRawDataToArm(batchItem, rawFilename, userAccount);

assertTrue(result);
}
Expand Down
24 changes: 14 additions & 10 deletions ...nTest/java/uk/gov/hmcts/darts/cases/controller/CaseControllerAdminGetCaseByIdIntTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,8 @@ void adminGetCaseById_WithCaseOpenAndReportingRestrictions() throws Exception {
String actualResponse = mvcResult.getResponse().getContentAsString();
String expectedResponse = getContentsFromFile(
"tests/cases/CaseControllerAdminGetCaseByIdTest/testCaseOpen/expectedResponse.json");
expectedResponse = expectedResponse.replace("<CREATED_AT>", courtCase.getCreatedDateTime().format(DateTimeFormatter.ISO_DATE_TIME));
expectedResponse = expectedResponse.replace("<LAST_MODIFIED_AT>", courtCase.getLastModifiedDateTime().format(DateTimeFormatter.ISO_DATE_TIME));
expectedResponse = expectedResponse.replace("<CREATED_AT>", formatUtc(courtCase.getCreatedDateTime()));
expectedResponse = expectedResponse.replace("<LAST_MODIFIED_AT>", formatUtc(courtCase.getLastModifiedDateTime()));
JSONAssert.assertEquals(expectedResponse, actualResponse, JSONCompareMode.NON_EXTENSIBLE);

}
Expand Down Expand Up @@ -176,8 +176,8 @@ void adminGetCaseById_WithCaseClosedAndReportingRestrictions() throws Exception
String actualResponse = mvcResult.getResponse().getContentAsString();
String expectedResponse = getContentsFromFile(
"tests/cases/CaseControllerAdminGetCaseByIdTest/testCaseClosed/expectedResponse.json");
expectedResponse = expectedResponse.replace("<CREATED_AT>", courtCase.getCreatedDateTime().format(DateTimeFormatter.ISO_DATE_TIME));
expectedResponse = expectedResponse.replace("<LAST_MODIFIED_AT>", courtCase.getLastModifiedDateTime().format(DateTimeFormatter.ISO_DATE_TIME));
expectedResponse = expectedResponse.replace("<CREATED_AT>", formatUtc(courtCase.getCreatedDateTime()));
expectedResponse = expectedResponse.replace("<LAST_MODIFIED_AT>", formatUtc(courtCase.getLastModifiedDateTime()));
JSONAssert.assertEquals(expectedResponse, actualResponse, JSONCompareMode.NON_EXTENSIBLE);

}
Expand Down Expand Up @@ -213,8 +213,8 @@ void adminGetCaseById_IsAnonymised() throws Exception {
String expectedResponse = getContentsFromFile(
"tests/cases/CaseControllerAdminGetCaseByIdTest/testIsAnonymised/expectedResponse.json");
expectedResponse = expectedResponse
.replace("<CREATED_AT>", courtCase.getCreatedDateTime().format(DateTimeFormatter.ISO_DATE_TIME))
.replace("<LAST_MODIFIED_AT>", courtCase.getLastModifiedDateTime().format(DateTimeFormatter.ISO_DATE_TIME))
.replace("<CREATED_AT>", formatUtc(courtCase.getCreatedDateTime()))
.replace("<LAST_MODIFIED_AT>", formatUtc(courtCase.getLastModifiedDateTime()))
.replace("<DATA_ANONYMISED_BY>", String.valueOf(anonymisedBy.getId()));
JSONAssert.assertEquals(expectedResponse, actualResponse, JSONCompareMode.NON_EXTENSIBLE);
}
Expand Down Expand Up @@ -248,8 +248,8 @@ void adminGetCaseById_IsDeleted() throws Exception {
String expectedResponse = getContentsFromFile(
"tests/cases/CaseControllerAdminGetCaseByIdTest/expectedResponse_isDeleted.json");
expectedResponse = expectedResponse
.replace("<CREATED_AT>", courtCase.getCreatedDateTime().format(DateTimeFormatter.ISO_DATE_TIME))
.replace("<LAST_MODIFIED_AT>", courtCase.getLastModifiedDateTime().format(DateTimeFormatter.ISO_DATE_TIME))
.replace("<CREATED_AT>", formatUtc(courtCase.getCreatedDateTime()))
.replace("<LAST_MODIFIED_AT>", formatUtc(courtCase.getLastModifiedDateTime()))
.replace("<CASE_DELETED_BY>", String.valueOf(deletedBy.getId()));
JSONAssert.assertEquals(expectedResponse, actualResponse, JSONCompareMode.NON_EXTENSIBLE);
}
Expand Down Expand Up @@ -299,8 +299,8 @@ void adminGetCaseById_WithRetentionAndAdditionalCaseDetails() throws Exception {
String expectedResponse = getContentsFromFile(
"tests/cases/CaseControllerAdminGetCaseByIdTest/expectedResponse_withRetentionAndAdditionalCaseDetails.json");
expectedResponse = expectedResponse
.replace("<CREATED_AT>", courtCase.getCreatedDateTime().format(DateTimeFormatter.ISO_DATE_TIME))
.replace("<LAST_MODIFIED_AT>", courtCase.getLastModifiedDateTime().format(DateTimeFormatter.ISO_DATE_TIME));
.replace("<CREATED_AT>", formatUtc(courtCase.getCreatedDateTime()))
.replace("<LAST_MODIFIED_AT>", formatUtc(courtCase.getLastModifiedDateTime()));
JSONAssert.assertEquals(expectedResponse, actualResponse, JSONCompareMode.NON_EXTENSIBLE);
}

Expand Down Expand Up @@ -328,4 +328,8 @@ private List<EventEntity> createEventsWithDifferentTimestamps(List<OffsetDateTim
return event;
}).toList();
}

private String formatUtc(OffsetDateTime offsetDateTime) {
return offsetDateTime.withOffsetSameInstant(UTC).format(DateTimeFormatter.ISO_DATE_TIME);
}
}
9 changes: 6 additions & 3 deletions ...tionTest/java/uk/gov/hmcts/darts/task/service/UnstructuredToArmBatchProcessorIntTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,7 +384,10 @@ void processUnstructuredToArm_ShouldMovePreviousArmFailedFromUnstructuredToArmSt
externalObjectDirectoryStub.createAndSaveEod(medias.get(4), STORED, UNSTRUCTURED);
externalObjectDirectoryStub.createAndSaveEod(medias.get(4), ARM_INGESTION, ARM, eod -> eod.setTransferAttempts(1));
externalObjectDirectoryStub.createAndSaveEod(medias.get(5), STORED, UNSTRUCTURED);
externalObjectDirectoryStub.createAndSaveEod(medias.get(5), ARM_RAW_DATA_PUSHED, ARM, eod -> eod.setTransferAttempts(1));
var armEod5 = externalObjectDirectoryStub.createAndSaveEod(medias.get(5), ARM_RAW_DATA_PUSHED, ARM, eod -> eod.setTransferAttempts(1));

when(armDataManagementApi.listSubmissionBlobs(armEod5.getId() + "_"))
.thenReturn(List.of(format("%d_%d_1", armEod5.getId(), medias.get(5).getId())));

//when
unstructuredToArmProcessor.processUnstructuredToArm(10);
Expand All @@ -403,7 +406,7 @@ void processUnstructuredToArm_ShouldMovePreviousArmFailedFromUnstructuredToArmSt
var armDropzoneEodsMedia4 = eodRepository.findByMediaStatusAndType(medias.get(4), EodHelper.armDropZoneStatus(), EodHelper.armLocation());
assertThat(armDropzoneEodsMedia4).hasSize(1);
var armDropzoneEodsMedia5 = eodRepository.findByMediaStatusAndType(medias.get(5), EodHelper.armDropZoneStatus(), EodHelper.armLocation());
assertThat(armDropzoneEodsMedia5).hasSize(0);
assertThat(armDropzoneEodsMedia5).hasSize(1);

verify(archiveRecordFileGenerator).generateArchiveRecords(manifestFileNameCaptor.capture(), any());
String manifestFileName = manifestFileNameCaptor.getValue();
Expand All @@ -415,7 +418,7 @@ void processUnstructuredToArm_ShouldMovePreviousArmFailedFromUnstructuredToArmSt
ArgumentCaptor<String> manifestFileContentCaptor = ArgumentCaptor.forClass(String.class);
verify(dataStoreToArmHelper).convertStringToBinaryData(manifestFileContentCaptor.capture());
String manifestFileContent = manifestFileContentCaptor.getValue();
assertThat(manifestFileContent.lines().count()).isEqualTo(6);
assertThat(manifestFileContent.lines().count()).isEqualTo(8);
assertThat(manifestFileContent).contains(
format("_%d_", medias.getFirst().getId()),
format("_%d_", medias.get(1).getId())
Expand Down
21 changes: 19 additions & 2 deletions src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/ArmServiceStubImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,56 +23,73 @@ public String saveBlobData(String containerName, String filename, BinaryData bin
@Override
public String saveBlobData(String containerName, BinaryData binaryData, String blobPathAndName) {
logStubUsageWarning();
log.warn("Returning filename to mimic successful upload: {}", blobPathAndName);
log.warn("Saving blob data: {}", blobPathAndName);
return blobPathAndName;
}

@Override
public List<String> listSubmissionBlobs(String containerName, String filename) {
logStubUsageWarning();
log.warn("Listing submission blobs: {}", filename);
List<String> listedBlobs = new ArrayList<>();
listedBlobs.add(filename);
return listedBlobs;
}

@Override
public List<String> listResponseBlobs(String containerName, String filename) {
logStubUsageWarning();
log.warn("Listing response blobs: {}", filename);
List<String> listedBlobs = new ArrayList<>();
listedBlobs.add(filename);
return listedBlobs;
}

@Override
public List<String> listSubmissionBlobsUsingBatch(String containerName, String filename, Integer batchSize) {
logStubUsageWarning();
log.warn("Listing submission blobs: {}", filename);
List<String> listedBlobs = new ArrayList<>();
listedBlobs.add(filename);
return listedBlobs;
}

@Override
public List<String> listResponseBlobsUsingBatch(String containerName, String filename, Integer batchSize) {
logStubUsageWarning();
log.warn("Listing response blobs: {}", filename);
List<String> listedBlobs = new ArrayList<>();
listedBlobs.add(filename);
return listedBlobs;
}

@Override
public ContinuationTokenBlobs listResponseBlobsWithMarker(String containerName, String filename, Integer batchSize, String continuationToken) {
return ContinuationTokenBlobs.builder().build();
logStubUsageWarning();
log.warn("Listing response blobs: {}", filename);
List<String> listedBlobs = new ArrayList<>();
listedBlobs.add(filename);
return ContinuationTokenBlobs.builder().blobNamesAndPaths(listedBlobs).build();
}

@Override
public ContinuationTokenBlobs listSubmissionBlobsWithMarker(String containerName, String filename, Integer batchSize, String continuationToken) {
logStubUsageWarning();
log.warn("Listing submission blobs: {}", filename);
return ContinuationTokenBlobs.builder().build();
}

@Override
public BinaryData getBlobData(String containerName, String blobName) {
logStubUsageWarning();
log.warn("Getting stub blob data: {}", blobName);
return BinaryData.fromBytes(new byte[1024]);
}

@Override
public boolean deleteBlobData(String containerName, String blobPathAndName) {
logStubUsageWarning();
log.warn("Deleting blob called with: {}", blobPathAndName);
return true;
}

Expand Down
2 changes: 2 additions & 0 deletions src/main/java/uk/gov/hmcts/darts/arm/api/ArmDataManagementApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ public interface ArmDataManagementApi extends BlobContainerDownloadable {

List<String> listResponseBlobs(String prefix);

List<String> listSubmissionBlobs(String prefix);

List<String> listResponseBlobsUsingBatch(String prefix, int batchSize);

ContinuationTokenBlobs listResponseBlobsUsingMarker(String prefix, int batchSize, String continuationToken);
Expand Down
7 changes: 7 additions & 0 deletions src/main/java/uk/gov/hmcts/darts/arm/api/impl/ArmDataManagementApiImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@

@Service
@RequiredArgsConstructor
@SuppressWarnings({"PMD.TooManyMethods"})
public class ArmDataManagementApiImpl implements ArmDataManagementApi {

private final ArmService armService;
Expand Down Expand Up @@ -60,6 +61,12 @@ public List<String> listResponseBlobs(String prefix) {
return armService.listResponseBlobs(armDataManagementConfiguration.getContainerName(), prefix);
}

@Override
public List<String> listSubmissionBlobs(String prefix) {
return armService.listSubmissionBlobs(armDataManagementConfiguration.getContainerName(), prefix);
}


@Override
public List<String> listResponseBlobsUsingBatch(String prefix, int batchSize) {
return armService.listResponseBlobsUsingBatch(armDataManagementConfiguration.getContainerName(),
Expand Down
52 changes: 42 additions & 10 deletions src/main/java/uk/gov/hmcts/darts/arm/helper/DataStoreToArmHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,17 @@
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Component;
import uk.gov.hmcts.darts.arm.api.ArmDataManagementApi;
import uk.gov.hmcts.darts.arm.component.ArchiveRecordFileGenerator;
import uk.gov.hmcts.darts.arm.config.ArmDataManagementConfiguration;
import uk.gov.hmcts.darts.arm.model.ArchiveRecord;
import uk.gov.hmcts.darts.arm.model.batch.ArmBatchItem;
import uk.gov.hmcts.darts.arm.model.batch.ArmBatchItems;
import uk.gov.hmcts.darts.arm.service.ArchiveRecordService;
import uk.gov.hmcts.darts.common.entity.ExternalLocationTypeEntity;
import uk.gov.hmcts.darts.common.entity.ExternalObjectDirectoryEntity;
import uk.gov.hmcts.darts.common.entity.ObjectRecordStatusEntity;
Expand All @@ -31,6 +34,7 @@
import java.util.Optional;
import java.util.UUID;

import static java.lang.String.format;
import static java.util.Objects.nonNull;
import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.ARM_INGESTION;
import static uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum.ARM_RAW_DATA_FAILED;
Expand All @@ -39,15 +43,18 @@
@Component
@Slf4j
@RequiredArgsConstructor
@SuppressWarnings({"PMD.TooManyMethods", "PMD.AvoidInstanceofChecksInCatchClause"})
@SuppressWarnings({"PMD.TooManyMethods", "PMD.AvoidInstanceofChecksInCatchClause", "PMD.GodClass"})
public class DataStoreToArmHelper {

private static final int BLOB_ALREADY_EXISTS_STATUS_CODE = 409;
private static final String SUBMISSION_FILE_SEPARATOR = "_";

private final ExternalObjectDirectoryRepository externalObjectDirectoryRepository;
private final ArmDataManagementConfiguration armDataManagementConfiguration;
private final LogApi logApi;
private final ArmDataManagementApi armDataManagementApi;
private final ArchiveRecordFileGenerator archiveRecordFileGenerator;
private final ArchiveRecordService archiveRecordService;

public List<Long> getEodEntitiesToSendToArm(ExternalLocationTypeEntity sourceLocation,
ExternalLocationTypeEntity armLocation, int maxResultSize) {
Expand Down Expand Up @@ -77,7 +84,6 @@ public List<Long> getEodEntitiesToSendToArm(ExternalLocationTypeEntity sourceLoc
return returnList;
}


public Optional<ExternalObjectDirectoryEntity> getExternalObjectDirectoryEntity(
ExternalObjectDirectoryEntity externalObjectDirectoryEntity, ExternalLocationTypeEntity eodSourceLocation, ObjectRecordStatusEntity status) {

Expand Down Expand Up @@ -164,7 +170,7 @@ public String generateRawFilename(ExternalObjectDirectoryEntity externalObjectDi
documentId = externalObjectDirectoryEntity.getCaseDocument().getId();
}

return String.format("%s_%s_%s", entityId, documentId, transferAttempts);
return format("%s_%s_%s", entityId, documentId, transferAttempts);
}

@SneakyThrows
Expand Down Expand Up @@ -215,10 +221,10 @@ public void updateExternalObjectDirectoryFailedTransferAttempts(ExternalObjectDi

public String getArchiveRecordsFileName(String manifestFilePrefix) {
String fileNameFormat = "%s_%s.%s";
return String.format(fileNameFormat,
manifestFilePrefix,
UUID.randomUUID().toString(),
armDataManagementConfiguration.getFileExtension()
return format(fileNameFormat,
manifestFilePrefix,
UUID.randomUUID(),
armDataManagementConfiguration.getFileExtension()
);
}

Expand Down Expand Up @@ -256,10 +262,36 @@ public ExternalObjectDirectoryEntity createArmEodWithArmIngestionStatus(External
return savedArmEod;
}

public boolean shouldPushRawDataToArm(ArmBatchItem batchItem) {
return equalsAnyStatus(batchItem.getPreviousStatus(), EodHelper.armIngestionStatus(), EodHelper.failedArmRawDataStatus());
}
public boolean shouldPushRawDataToArm(ArmBatchItem batchItem, String rawFilename, UserAccountEntity userAccount) {

if (equalsAnyStatus(batchItem.getPreviousStatus(), EodHelper.armIngestionStatus(), EodHelper.failedArmRawDataStatus())) {
return true;
} else {
ExternalObjectDirectoryEntity armEod = batchItem.getArmEod();
String prefix = format("%d_", armEod.getId());
List<String> submissionBlobs = armDataManagementApi.listSubmissionBlobs(prefix);

if (CollectionUtils.isNotEmpty(submissionBlobs)) {
batchItem.setRawFilePushSuccessful(true);

ArchiveRecord archiveRecord = archiveRecordService.generateArchiveRecordInfo(batchItem.getArmEod().getId(), rawFilename);
batchItem.setArchiveRecord(archiveRecord);
updateExternalObjectDirectoryStatus(armEod, EodHelper.armRawDataPushedStatus(), userAccount);
int index = submissionBlobs.getFirst().lastIndexOf(SUBMISSION_FILE_SEPARATOR);
if (index != -1 && index + 1 < submissionBlobs.getFirst().length()) {
Integer attempts = Integer.parseInt(submissionBlobs.getFirst().substring(index + 1));
armEod.setTransferAttempts(attempts);
externalObjectDirectoryRepository.saveAndFlush(armEod);
}
return false;
} else {
updateExternalObjectDirectoryStatusToFailed(armEod, EodHelper.failedArmRawDataStatus(), userAccount);
batchItem.setRawFilePushSuccessful(false);
batchItem.undoManifestFileChange();
return true;
}
}
}

public boolean shouldAddEntryToManifestFile(ArmBatchItem batchItem) {
return equalsAnyStatus(batchItem.getPreviousStatus(), EodHelper.failedArmManifestFileStatus(), EodHelper.armResponseManifestFailedStatus());
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/uk/gov/hmcts/darts/arm/service/impl/DetsToArmBatchPushProcessorImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ private void createAndSendBatchFile(List<ExternalObjectDirectoryEntity> eodsForB

String rawFilename = dataStoreToArmHelper.generateRawFilename(armEod);

if (dataStoreToArmHelper.shouldPushRawDataToArm(batchItem)) {
if (dataStoreToArmHelper.shouldPushRawDataToArm(batchItem, rawFilename, userAccount)) {
pushRawDataAndCreateArchiveRecordIfSuccess(batchItem, rawFilename, userAccount, objectStateRecord);
} else if (dataStoreToArmHelper.shouldAddEntryToManifestFile(batchItem)) {
batchItem.setArchiveRecord(archiveRecordService.generateArchiveRecordInfo(batchItem.getArmEod().getId(), rawFilename));
Expand Down
Loading
Loading