Skip to content

Commit 625b469

Browse files
marcaurelemarcaurele
committed
Live migration fixes (#20)
Multiple bugs fixes: * Incorrect disk size: when provisioning the volume at the destination, the size was taken from the template and not from the original volume itself. * Incorrect details when returning an error: now if an exception is raised by Libvirt at the agent level, the error message is forwarded correctly all the way into the job response text. * Incorrect volume state: when a migration failed, the volume state was inconsistent and stayed in "Migrating", those interfering with future commands on the VM. As a result, the event message wasn't sent either.
1 parent 2f310e5 commit 625b469

File tree

11 files changed

+67
-63
lines changed

11 files changed

+67
-63
lines changed

‎engine/api/src/com/cloud/vm/VirtualMachineManager.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import java.util.List;
2222
import java.util.Map;
2323

24+
import com.cloud.exception.VirtualMachineMigrationException;
2425
import org.apache.cloudstack.framework.config.ConfigKey;
2526

2627
import com.cloud.agent.api.to.NicTO;
@@ -114,7 +115,7 @@ void orchestrateStart(String vmUuid, Map<VirtualMachineProfile.Param, Object> pa
114115

115116
void migrate(String vmUuid, long srcHostId, DeployDestination dest) throws ResourceUnavailableException, ConcurrentOperationException;
116117

117-
void migrateWithStorage(String vmUuid, long srcId, long destId, Map<Long, Long> volumeToPool) throws ResourceUnavailableException, ConcurrentOperationException;
118+
void migrateWithStorage(String vmUuid, long srcId, long destId, Map<Long, Long> volumeToPool) throws ResourceUnavailableException, ConcurrentOperationException, VirtualMachineMigrationException;
118119

119120
void reboot(String vmUuid, Map<VirtualMachineProfile.Param, Object> params) throws InsufficientCapacityException, ResourceUnavailableException;
120121

‎engine/api/src/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import java.util.Map;
2222
import java.util.Set;
2323

24+
import com.cloud.exception.VirtualMachineMigrationException;
2425
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
2526
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
2627
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
@@ -102,7 +103,7 @@ VolumeInfo moveVolume(VolumeInfo volume, long destPoolDcId, Long destPoolPodId,
102103

103104
void revokeAccess(long vmId, long hostId);
104105

105-
void migrateVolumes(VirtualMachine vm, VirtualMachineTO vmTo, Host srcHost, Host destHost, Map<Volume, StoragePool> volumeToPool);
106+
void liveMigrateVolumes(VirtualMachine vm, VirtualMachineTO vmTo, Host srcHost, Host destHost, Map<Volume, StoragePool> volumeToPool) throws VirtualMachineMigrationException;
106107

107108
boolean storageMigration(VirtualMachineProfile vm, StoragePool destPool) throws StorageUnavailableException;
108109

‎engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/CopyCommandResult.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,4 +39,10 @@ public String getPath() {
3939
public Answer getAnswer() {
4040
return this.answer;
4141
}
42+
43+
@Override
44+
public String toString() {
45+
String sup = super.toString();
46+
return sup.substring(0, sup.length()-1) + ", path: " + (path == null ? "<null>" : path) + ", answer: " + (answer == null ? "<null>" : answer) + "}";
47+
}
4248
}

‎engine/api/src/org/apache/cloudstack/storage/command/CommandResult.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,4 +59,9 @@ public void setResult(String result) {
5959
this.success = false;
6060
}
6161
}
62+
63+
@Override
64+
public String toString() {
65+
return "CommandResult={success: " + success + ", aborted: " + aborted + ", result: " + (result == null ? "<null>" : result) + "}";
66+
}
6267
}

‎engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@
3838
import javax.naming.ConfigurationException;
3939

4040
import com.cloud.agent.api.AttachOrDettachConfigDriveCommand;
41+
import com.cloud.exception.VirtualMachineMigrationException;
42+
4143
import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
4244
import org.apache.cloudstack.context.CallContext;
4345
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
@@ -798,7 +800,7 @@ public void advanceStart(final String vmUuid, final Map<VirtualMachineProfile.Pa
798800
} catch (final InterruptedException e) {
799801
throw new RuntimeException("Operation is interrupted", e);
800802
} catch (final java.util.concurrent.ExecutionException e) {
801-
throw new RuntimeException("Execution excetion", e);
803+
throw new RuntimeException("Execution exception", e);
802804
}
803805

804806
final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob());
@@ -2148,7 +2150,7 @@ private <T extends VMInstanceVO> void moveVmOutofMigratingStateOnSuccess(final T
21482150

21492151
@Override
21502152
public void migrateWithStorage(final String vmUuid, final long srcHostId, final long destHostId, final Map<Long, Long> volumeToPool)
2151-
throws ResourceUnavailableException, ConcurrentOperationException {
2153+
throws ResourceUnavailableException, ConcurrentOperationException, VirtualMachineMigrationException {
21522154

21532155
final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext();
21542156
if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) {
@@ -2192,7 +2194,7 @@ public void migrateWithStorage(final String vmUuid, final long srcHostId, final
21922194
}
21932195

21942196
private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHostId, final long destHostId, final Map<Long, Long> volumeToPool) throws ResourceUnavailableException,
2195-
ConcurrentOperationException {
2197+
ConcurrentOperationException, VirtualMachineMigrationException {
21962198

21972199
final VMInstanceVO vm = _vmDao.findByUuid(vmUuid);
21982200

@@ -2204,8 +2206,7 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo
22042206
final HostPodVO pod = _podDao.findById(destHost.getPodId());
22052207
final Cluster cluster = _clusterDao.findById(destHost.getClusterId());
22062208
final DeployDestination destination = new DeployDestination(dc, pod, cluster, destHost);
2207-
2208-
VirtualMachineProfile vmSrc = new VirtualMachineProfileImpl(vm);
2209+
final VirtualMachineProfile vmSrc = new VirtualMachineProfileImpl(vm);
22092210
for (NicProfile nic : _networkMgr.getNicProfiles(vm)) {
22102211
vmSrc.addNic(nic);
22112212
}
@@ -2288,7 +2289,7 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo
22882289
}
22892290

22902291
// Migrate the vm and its volume.
2291-
volumeMgr.migrateVolumes(vm, to, srcHost, destHost, volumeToPoolMap);
2292+
volumeMgr.liveMigrateVolumes(vm, to, srcHost, destHost, volumeToPoolMap);
22922293
migrated = true;
22932294

22942295
try {
@@ -2325,10 +2326,13 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo
23252326

23262327
volumeMgr.confirmMigration(profile, srcHostId, destHostId, migrated);
23272328

2328-
23292329
work.setStep(Step.Done);
23302330
_workDao.update(work.getId(), work);
23312331
}
2332+
2333+
if (!migrated) {
2334+
throw new VirtualMachineMigrationException("Migration failed");
2335+
}
23322336
}
23332337

23342338
@Override

‎engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java‎

Lines changed: 3 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import javax.inject.Inject;
3131
import javax.naming.ConfigurationException;
3232

33+
import com.cloud.exception.VirtualMachineMigrationException;
3334
import com.cloud.storage.StorageManager;
3435
import com.cloud.storage.VMTemplateVO;
3536
import com.cloud.storage.dao.VMTemplateDao;
@@ -980,28 +981,8 @@ public Volume migrateVolume(Volume volume, StoragePool destPool) throws StorageU
980981
}
981982
}
982983

983-
@DB
984-
protected Volume liveMigrateVolume(Volume volume, StoragePool destPool) {
985-
VolumeInfo vol = volFactory.getVolume(volume.getId());
986-
AsyncCallFuture<VolumeApiResult> future = volService.migrateVolume(vol, (DataStore)destPool);
987-
try {
988-
VolumeApiResult result = future.get();
989-
if (result.isFailed()) {
990-
s_logger.debug("migrate volume failed:" + result.getResult());
991-
return null;
992-
}
993-
return result.getVolume();
994-
} catch (InterruptedException e) {
995-
s_logger.debug("migrate volume failed", e);
996-
return null;
997-
} catch (ExecutionException e) {
998-
s_logger.debug("migrate volume failed", e);
999-
return null;
1000-
}
1001-
}
1002-
1003984
@Override
1004-
public void migrateVolumes(VirtualMachine vm, VirtualMachineTO vmTo, Host srcHost, Host destHost, Map<Volume, StoragePool> volumeToPool) {
985+
public void liveMigrateVolumes(VirtualMachine vm, VirtualMachineTO vmTo, Host srcHost, Host destHost, Map<Volume, StoragePool> volumeToPool) throws VirtualMachineMigrationException {
1005986
// Check if all the vms being migrated belong to the vm.
1006987
// Check if the storage pool is of the right type.
1007988
// Create a VolumeInfo to DataStore map too.
@@ -1027,7 +1008,7 @@ public void migrateVolumes(VirtualMachine vm, VirtualMachineTO vmTo, Host srcHos
10271008
CommandResult result = future.get();
10281009
if (result.isFailed()) {
10291010
s_logger.debug("Failed to migrate vm " + vm + " along with its volumes. " + result.getResult());
1030-
throw new CloudRuntimeException("Failed to migrate vm " + vm + " along with its volumes.");
1011+
throw new VirtualMachineMigrationException("Failed to migrate vm " + vm + " along with its volumes.");
10311012
}
10321013
} catch (InterruptedException e) {
10331014
s_logger.debug("Failed to migrated vm " + vm + " along with its volumes.", e);

‎engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java‎

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1573,29 +1573,23 @@ protected Void migrateVmWithVolumesCallBack(AsyncCallbackDispatcher<VolumeServic
15731573
CopyCommandResult result = callback.getResult();
15741574
AsyncCallFuture<CommandResult> future = context.future;
15751575
CommandResult res = new CommandResult();
1576-
try {
1577-
if (result.isFailed()) {
1578-
MigrateWithStorageAnswer answer = (MigrateWithStorageAnswer)result.getAnswer();
1579-
res.setSuccess(false);
1576+
1577+
if (result.isFailed()) {
1578+
res.setSuccess(false);
1579+
MigrateWithStorageAnswer answer = (MigrateWithStorageAnswer)result.getAnswer();
1580+
if (answer != null) {
15801581
res.setAborted(answer.isAborted());
1581-
res.setResult(result.getResult());
1582-
for (Map.Entry<VolumeInfo, DataStore> entry : volumeToPool.entrySet()) {
1583-
VolumeInfo volume = entry.getKey();
1584-
volume.processEvent(Event.OperationFailed);
1585-
}
1586-
} else {
1587-
for (Map.Entry<VolumeInfo, DataStore> entry : volumeToPool.entrySet()) {
1588-
VolumeInfo volume = entry.getKey();
1589-
volume.processEvent(Event.OperationSuccessed);
1590-
}
15911582
}
1592-
future.complete(res);
1593-
} catch (Exception e) {
1594-
s_logger.error("Failed to process migration volume callback", e);
1595-
res.setResult(e.toString());
1596-
future.complete(res);
1583+
res.setResult(result.getResult());
15971584
}
15981585

1586+
for (Map.Entry<VolumeInfo, DataStore> entry : volumeToPool.entrySet()) {
1587+
VolumeInfo volume = entry.getKey();
1588+
volume.processEvent(result.isFailed() || result.isAborted() ? Event.OperationFailed : Event.OperationSuccessed);
1589+
}
1590+
1591+
future.complete(res);
1592+
15991593
return null;
16001594
}
16011595

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateCommandWrapper.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,10 @@ public Answer execute(final CreateCommand command, final LibvirtComputingResourc
5959
vol = libvirtComputingResource.templateToPrimaryDownload(command.getTemplateUrl(), primaryPool, dskch.getPath());
6060
} else {
6161
baseVol = primaryPool.getPhysicalDisk(command.getTemplateUrl());
62-
vol = storagePoolMgr.createDiskFromTemplate(baseVol, dskch.getPath(), dskch.getProvisioningType(), primaryPool, 0);
62+
if (baseVol.getSize() > disksize) {
63+
disksize = baseVol.getSize();
64+
}
65+
vol = storagePoolMgr.createDiskFromTemplate(baseVol, dskch.getPath(), dskch.getProvisioningType(), primaryPool, disksize, 0);
6366
}
6467
if (vol == null) {
6568
return new Answer(command, false, " Can't create storage volume on storage pool");

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -336,11 +336,6 @@ public boolean deleteStoragePool(StoragePoolType type, String uuid) {
336336
return true;
337337
}
338338

339-
public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, Storage.ProvisioningType provisioningType,
340-
KVMStoragePool destPool, int timeout) {
341-
return createDiskFromTemplate(template, name, provisioningType, destPool, template.getSize(), timeout);
342-
}
343-
344339
public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, Storage.ProvisioningType provisioningType,
345340
KVMStoragePool destPool, long size, int timeout) {
346341
StorageAdaptor adaptor = getStorageAdaptor(destPool.getType());

‎plugins/hypervisors/kvm/src/org/apache/cloudstack/storage/motion/KVMStorageMotionStrategy.java‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -105,21 +105,31 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeMap, VirtualMachineTO vmT
105105
if (instance != null) {
106106
answer = migrateVmWithVolumes(instance, vmTo, srcHost, destHost, volumeMap);
107107
errMsg = answer.getDetails();
108+
109+
if (s_logger.isDebugEnabled()) {
110+
s_logger.debug("Got answer from migration, result: " + (answer.getResult() ? "ok" : "failed, reason: " + (answer.getDetails() == null ? "<null>" : answer.getDetails())));
111+
}
112+
108113
} else {
109114
throw new CloudRuntimeException("Unsupported operation requested for moving data.");
110115
}
111116
} catch (Exception e) {
112117
s_logger.error("copyAsync failed", e);
113-
errMsg = e.toString();
118+
errMsg = e.getMessage();
114119
}
115120

116-
if (s_logger.isDebugEnabled()) {
117-
s_logger.debug("Got answer from migration, result: " + (answer.getResult() ? "ok" : "failed, reason: " + answer.getDetails()));
121+
CopyCommandResult result = new CopyCommandResult("", answer);
122+
123+
if (answer != null) {
124+
result.setSuccess(answer.getResult());
125+
if (!answer.getResult()) {
126+
result.setResult(errMsg);
127+
}
128+
} else {
129+
result.setSuccess(false);
130+
result.setResult(errMsg);
118131
}
119132

120-
CopyCommandResult result = new CopyCommandResult(null, answer);
121-
result.setResult(errMsg);
122-
result.setSuccess(answer.getResult());
123133
callback.complete(result);
124134
}
125135

@@ -145,6 +155,10 @@ private Answer migrateVmWithVolumes(VMInstanceVO vm, VirtualMachineTO to, Host s
145155
StoragePool destStoragePool = storageManager.findLocalStorageOnHost(destHost.getId());
146156
TemplateInfo templateImage = tmplFactory.getTemplate(rootVolume.getTemplateId(), DataStoreRole.Image);
147157

158+
if (s_logger.isDebugEnabled()) {
159+
s_logger.debug("Provisioning disk " + diskProfile.toString() + " on destination host");
160+
}
161+
148162
try {
149163
CreateCommand provisioningCommand = new CreateCommand(diskProfile, templateImage.getUuid(), destStoragePool, true);
150164
CreateAnswer provisioningAnwer = (CreateAnswer) agentMgr.send(destHost.getId(), provisioningCommand);

0 commit comments

Comments
 (0)