Merge branch 'fix-incremental-snapshot-migration' into '4.20.0.0-scclouds'

Adição de suporte à migração de snapshots e backups incrementais com KVM

Closes #2888

See merge request scclouds/scclouds!1248

Author: Fabrício Duarte

core/src/main/java/com/cloud/agent/api/MigrateBackupsBetweenSecondaryStoragesCommand.java

@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.DataTO;
+
+import java.util.List;
+
+public class MigrateBackupsBetweenSecondaryStoragesCommand extends MigrateBetweenSecondaryStoragesCommand {
+
+    List<List<DataTO>> backupChain;
+
+    public MigrateBackupsBetweenSecondaryStoragesCommand() {
+    }
+
+    public MigrateBackupsBetweenSecondaryStoragesCommand(List<List<DataTO>> backupChain, DataStoreTO srcDataStore, DataStoreTO destDataStore) {
+        super(srcDataStore, destDataStore);
+        this.backupChain = backupChain;
+    }
+
+    public List<List<DataTO>> getBackupChain() {
+        return backupChain;
+    }
+}

core/src/main/java/com/cloud/agent/api/MigrateBetweenSecondaryStoragesCommand.java

@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+import com.cloud.agent.api.to.DataStoreTO;
+
+public abstract class MigrateBetweenSecondaryStoragesCommand extends Command {
+
+    DataStoreTO srcDataStore;
+    DataStoreTO destDataStore;
+
+    public MigrateBetweenSecondaryStoragesCommand() {
+    }
+
+    public MigrateBetweenSecondaryStoragesCommand(DataStoreTO srcDataStore, DataStoreTO destDataStore) {
+        this.srcDataStore = srcDataStore;
+        this.destDataStore = destDataStore;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+
+    public DataStoreTO getSrcDataStore() {
+        return srcDataStore;
+    }
+
+    public DataStoreTO getDestDataStore() {
+        return destDataStore;
+    }
+}

core/src/main/java/com/cloud/agent/api/MigrateBetweenSecondaryStoragesCommandAnswer.java

@@ -0,0 +1,41 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+import com.cloud.utils.Pair;
+
+import java.util.List;
+
+public class MigrateBetweenSecondaryStoragesCommandAnswer extends Answer {
+
+    List<Pair<Long, String>> migratedResourcesIdAndCheckpointPath;
+
+    public MigrateBetweenSecondaryStoragesCommandAnswer() {
+    }
+
+    public MigrateBetweenSecondaryStoragesCommandAnswer(MigrateBetweenSecondaryStoragesCommand cmd, boolean success, String result, List<Pair<Long, String>> migratedResourcesIdAndCheckpointPath) {
+        super(cmd, success, result);
+        this.migratedResourcesIdAndCheckpointPath = migratedResourcesIdAndCheckpointPath;
+    }
+
+    public List<Pair<Long, String>> getMigratedResources() {
+        return migratedResourcesIdAndCheckpointPath;
+    }
+}

core/src/main/java/com/cloud/agent/api/MigrateSnapshotsBetweenSecondaryStoragesCommand.java

@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.DataTO;
+
+import java.util.List;
+import java.util.Set;
+
+public class MigrateSnapshotsBetweenSecondaryStoragesCommand extends MigrateBetweenSecondaryStoragesCommand {
+
+    List<DataTO> snapshotChain;
+    Set<Long> snapshotsIdToMigrate;
+
+    public MigrateSnapshotsBetweenSecondaryStoragesCommand() {
+    }
+
+    public MigrateSnapshotsBetweenSecondaryStoragesCommand(List<DataTO> snapshotChain, DataStoreTO srcDataStore, DataStoreTO destDataStore, Set<Long> snapshotsIdToMigrate) {
+        super(srcDataStore, destDataStore);
+        this.snapshotChain = snapshotChain;
+        this.snapshotsIdToMigrate = snapshotsIdToMigrate;
+    }
+
+    public List<DataTO> getSnapshotChain() {
+        return snapshotChain;
+    }
+
+    public Set<Long> getSnapshotsIdToMigrate() {
+        return snapshotsIdToMigrate;
+    }
+}

core/src/main/java/org/apache/cloudstack/storage/to/BackupDeltaTO.java

@@ -43,6 +43,11 @@ public BackupDeltaTO(DataStoreTO dataStoreTO, Hypervisor.HypervisorType hypervis
         this.format = Storage.ImageFormat.QCOW2;
     }
 
+    public BackupDeltaTO(long id, DataStoreTO dataStoreTO, Hypervisor.HypervisorType hypervisorType, String path) {
+        this(dataStoreTO, hypervisorType, path);
+        this.id = id;
+    }
+
     @Override
     public DataObjectType getObjectType() {
         return DataObjectType.BACKUP;

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java

@@ -30,6 +30,8 @@ public interface SnapshotInfo extends DataObject, Snapshot {
 
     SnapshotInfo getParent();
 
+    List<SnapshotInfo> getParents();
+
     String getPath();
 
     SnapshotInfo getChild();
@@ -40,6 +42,8 @@ public interface SnapshotInfo extends DataObject, Snapshot {
 
     List<SnapshotInfo> getChildren();
 
+    List<SnapshotInfo> getChildAndGrandchildren();
+
     VolumeInfo getBaseVolume();
 
     void addPayload(Object data);

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -213,6 +213,9 @@ public interface StorageManager extends StorageService {
             "Storage", "true", "Allow SSVMs to try copying public templates from one secondary storage to another instead of downloading them from the source.",
             true, ConfigKey.Scope.Zone, null);
 
+    ConfigKey<Integer> AgentMaxDataMigrationWaitTime = new ConfigKey<>("Advanced", Integer.class, "agent.max.data.migration.wait.time", "3600",
+            "The maximum time (in seconds) that the secondary storage data migration command sent to the KVM Agent will be executed before a timeout occurs.", true, ConfigKey.Scope.Cluster);
+
     /**
      * should we execute in sequence not involving any storages?
      * @return tru if commands should execute in sequence