diff --git a/plugins/backup/commvault/src/main/java/org/apache/cloudstack/backup/CommvaultBackupProvider.java b/plugins/backup/commvault/src/main/java/org/apache/cloudstack/backup/CommvaultBackupProvider.java index cff8b757f8ba..57cf988a3ae2 100644 --- a/plugins/backup/commvault/src/main/java/org/apache/cloudstack/backup/CommvaultBackupProvider.java +++ b/plugins/backup/commvault/src/main/java/org/apache/cloudstack/backup/CommvaultBackupProvider.java @@ -89,6 +89,7 @@ import java.util.stream.Collectors; import java.util.Collections; import java.util.Comparator; +import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.inject.Inject; @@ -104,6 +105,7 @@ public class CommvaultBackupProvider extends AdapterBase implements BackupProvid private static final int BASE_MT = 89; private static final Pattern VERSION_PATTERN = Pattern.compile("^(\\d+)\\s*SP\\s*(\\d+)(?:\\.(\\d+))?$", Pattern.CASE_INSENSITIVE); private static final String COMMVAULT_DIRECTORY = "/tmp/mold/backup"; + private static final long STALE_BACKUP_THRESHOLD_MS = TimeUnit.DAYS.toMillis(1); public ConfigKey CommvaultUrl = new ConfigKey<>("Advanced", String.class, "backup.plugin.commvault.url", "https://localhost/commandcenter/api", @@ -860,6 +862,11 @@ public String getDescription() { return "Commvault Backup Plugin"; } + private boolean isBackupManagedByThisProvider(Backup backup) { + BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); + return offering != null && Objects.equals(getName(), offering.getProvider()); + } + @Override public String getConfigComponentName() { return BackupService.class.getSimpleName(); @@ -874,6 +881,21 @@ public void syncBackups(VirtualMachine vm) { } final CommvaultClient client = getClient(vm.getDataCenterId()); for (final Backup backup: backupDao.listByVmId(vm.getDataCenterId(), vm.getId())) { + if (!isBackupManagedByThisProvider(backup)) { + continue; + } + if (Backup.Status.BackingUp.equals(backup.getStatus()) && isOlderThanOneDay(backup.getDate())) { + LOG.warn("Removing stale Commvault backup [{}] for VM [{}] stuck in BackingUp for over one day. External ID: [{}]", + backup.getUuid(), vm.getInstanceName(), backup.getExternalId()); + try { + if (deleteBackup(backup, true)) { + backupDao.remove(backup.getId()); + } + } catch (Exception e) { + LOG.warn("Failed to delete stale Commvault backup [{}] for VM [{}]", backup.getUuid(), vm.getInstanceName(), e); + } + continue; + } String externalId = backup.getExternalId(); String jobId = externalId.substring(externalId.lastIndexOf(',') + 1).trim(); String path = externalId.substring(0, externalId.lastIndexOf(',')); @@ -906,6 +928,10 @@ public void syncBackups(VirtualMachine vm) { return; } + private boolean isOlderThanOneDay(Date backupDate) { + return backupDate != null && backupDate.getTime() <= System.currentTimeMillis() - STALE_BACKUP_THRESHOLD_MS; + } + @Override public boolean checkBackupAgent(final Long zoneId) { Map checkResult = new HashMap<>(); diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 7ee7612230ee..d081e3502c60 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -50,6 +50,7 @@ import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.backup.dao.BackupRepositoryDao; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -72,6 +73,7 @@ import java.util.Locale; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.TimeUnit; import java.util.UUID; import java.util.stream.Collectors; @@ -79,6 +81,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Configurable { private static final Logger LOG = LogManager.getLogger(NASBackupProvider.class); + private static final long STALE_BACKUP_THRESHOLD_MS = TimeUnit.DAYS.toMillis(1); ConfigKey NASBackupRestoreMountTimeout = new ConfigKey<>("Advanced", Integer.class, "nas.backup.restore.mount.timeout", @@ -90,6 +93,9 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Inject private BackupDao backupDao; + @Inject + private BackupOfferingDao backupOfferingDao; + @Inject private BackupRepositoryDao backupRepositoryDao; @@ -481,7 +487,7 @@ public boolean deleteBackup(Backup backup, boolean forced) { } if (answer != null && answer.getResult()) { - return backupDao.remove(backup.getId()); + return true; } logger.debug("There was an error removing the backup with id {}", backup.getId()); @@ -600,6 +606,11 @@ public String getDescription() { return "NAS Backup Plugin"; } + private boolean isBackupManagedByThisProvider(Backup backup) { + BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); + return offering != null && Objects.equals(getName(), offering.getProvider()); + } + @Override public String getConfigComponentName() { return BackupService.class.getSimpleName(); @@ -607,6 +618,27 @@ public String getConfigComponentName() { @Override public void syncBackups(VirtualMachine vm) { + for (final Backup backup : backupDao.listByVmId(vm.getDataCenterId(), vm.getId())) { + if (!isBackupManagedByThisProvider(backup)) { + continue; + } + if (!(backup instanceof BackupVO) || !Backup.Status.BackingUp.equals(backup.getStatus()) || !isOlderThanOneDay(backup.getDate())) { + continue; + } + LOG.warn("Removing stale NAS backup [{}] for VM [{}] stuck in BackingUp for over one day. Repository path: [{}]", + backup.getUuid(), vm.getInstanceName(), backup.getExternalId()); + try { + if (deleteBackup(backup, true)) { + backupDao.remove(backup.getId()); + } + } catch (Exception e) { + LOG.warn("Failed to delete stale NAS backup [{}] for VM [{}]", backup.getUuid(), vm.getInstanceName(), e); + } + } + } + + private boolean isOlderThanOneDay(Date backupDate) { + return backupDate != null && backupDate.getTime() <= System.currentTimeMillis() - STALE_BACKUP_THRESHOLD_MS; } @Override @@ -621,4 +653,4 @@ public void syncBackups(VirtualMachine vm) { @Override public boolean updateBackupPlan(final Long zoneId, final String retentionPeriod, final String externalId) { return true; } -} \ No newline at end of file +}