Implement template replication up to capacity after upload and during template creation

Author: Damans227

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java

@@ -71,6 +71,8 @@ public TemplateInfo getTemplate() {
 
     boolean canCopyTemplateToImageStore(long templateId, long zoneId);
 
+    void replicateTemplateUpToCap(long templateId, long zoneId);
+
     void downloadBootstrapSysTemplate(DataStore store);
 
     void addSystemVMTemplatesToSecondary(DataStore store);

engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java

@@ -351,6 +351,48 @@ private boolean hasReachedSecStorageCopyLimit(VMTemplateVO template, long zoneId
         return !canCopyTemplateToImageStore(template.getId(), zoneId);
     }
 
+    @Override
+    public void replicateTemplateUpToCap(long templateId, long zoneId) {
+        VMTemplateVO template = _templateDao.findById(templateId);
+        if (template == null) {
+            return;
+        }
+        List<DataStore> stores = _storeMgr.getImageStoresByScope(new ZoneScope(zoneId));
+        if (stores == null || stores.isEmpty()) {
+            return;
+        }
+        for (DataStore store : stores) {
+            if (hasActiveTemplateCopyOnStore(templateId, store.getId())) {
+                continue;
+            }
+            if (!canCopyTemplateToImageStore(templateId, zoneId)) {
+                break;
+            }
+            try {
+                storageOrchestrator.orchestrateTemplateCopyFromSecondaryStores(templateId, store);
+            } catch (Exception e) {
+                logger.warn("Failed to proactively replicate template [{}] to image store [{}] in zone [{}]: {}",
+                        template.getUniqueName(), store.getName(), zoneId, e.getMessage());
+            }
+        }
+    }
+
+    private boolean hasActiveTemplateCopyOnStore(long templateId, long storeId) {
+        List<TemplateDataStoreVO> rows = _vmTemplateStoreDao.listByTemplateStore(templateId, storeId);
+        if (rows == null) {
+            return false;
+        }
+        for (TemplateDataStoreVO row : rows) {
+            State st = row.getState();
+            Status ds = row.getDownloadState();
+            if (st != State.Failed && st != State.Destroyed
+                    && ds != Status.ABANDONED && ds != Status.DOWNLOAD_ERROR) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     @Override
     public void enforceSecStorageCopyLimit(long templateId, long zoneId) {
         VMTemplateVO template = _templateDao.findById(templateId);

server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java

@@ -451,6 +451,12 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
                             if (logger.isDebugEnabled()) {
                                 logger.debug("Template {} uploaded successfully", tmpTemplate);
                             }
+                            try {
+                                templateService.replicateTemplateUpToCap(tmpTemplate.getId(), vo.getDataCenterId());
+                            } catch (Exception e) {
+                                logger.warn("Failed to schedule additional copies for uploaded template [{}] in zone [{}]: {}",
+                                        tmpTemplate.getUuid(), vo.getDataCenterId(), e.getMessage());
+                            }
                             break;
                         case IN_PROGRESS:
                             if (tmpTemplate.getState() == VirtualMachineTemplate.State.NotUploaded) {

server/src/main/java/com/cloud/template/TemplateManagerImpl.java

@@ -1722,10 +1722,10 @@ public VirtualMachineTemplate createPrivateTemplate(CreateTemplateCmd command) t
         Account caller = CallContext.current().getCallingAccount();
         boolean kvmSnapshotOnlyInPrimaryStorage = false;
         SnapshotInfo snapInfo = null;
+        long zoneId = 0;
 
         try {
             TemplateInfo tmplInfo = _tmplFactory.getTemplate(templateId, DataStoreRole.Image);
-            long zoneId = 0;
             if (snapshotId != null) {
                 snapshot = _snapshotDao.findById(snapshotId);
                 if (command.getZoneId() == null) {
@@ -1865,6 +1865,12 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
         }
 
         if (privateTemplate != null) {
+            try {
+                _tmpltSvr.replicateTemplateUpToCap(privateTemplate.getId(), zoneId);
+            } catch (Exception e) {
+                logger.warn("Failed to schedule additional copies for template [{}] in zone [{}]: {}",
+                        privateTemplate.getUniqueName(), zoneId, e.getMessage());
+            }
             return privateTemplate;
         } else {
             throw new CloudRuntimeException("Failed to create a Template");