Skip to content

Added hot plugging of vifs in case of VMware isolated networks #5912

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DaanHoogland merged 1 commit into from
Feb 4, 2022
Merged

Added hot plugging of vifs in case of VMware isolated networks #5912

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions api/src/main/java/com/cloud/agent/api/to/IpAddressTO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

import com.cloud.network.Networks.TrafficType;

import java.util.Map;

public class IpAddressTO {

private long accountId;
Expand All @@ -36,6 +38,8 @@ public class IpAddressTO {
private Integer nicDevId;
private boolean newNic;
private boolean isPrivateGateway;
private NicTO nicTO;
Map<String, String> details;

public IpAddressTO(long accountId, String ipAddress, boolean add, boolean firstIP, boolean sourceNat, String broadcastUri, String vlanGateway, String vlanNetmask,
String vifMacAddress, Integer networkRate, boolean isOneToOneNat) {
Expand Down Expand Up @@ -142,4 +146,21 @@ public boolean isPrivateGateway() {
public void setPrivateGateway(boolean isPrivateGateway) {
this.isPrivateGateway = isPrivateGateway;
}

public NicTO getNicTO() {
return nicTO;
}

public void setNicTO(NicTO nicTO) {
this.nicTO = nicTO;
}


public Map<String, String> getDetails() {
return details;
}

public void setDetails(Map<String, String> details) {
this.details = details;
}
}
4 changes: 2 additions & 2 deletions plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ private void configureDomainRouterNicsAndDetails(VirtualMachineProfile vm, Virtu
NicTO[] nics = to.getNics();

// reserve extra NICs
NicTO[] expandedNics = new NicTO[nics.length + vmwareMgr.getRouterExtraPublicNics()];
NicTO[] expandedNics = new NicTO[nics.length];
int i = 0;
int deviceId = -1;
for (i = 0; i < nics.length; i++) {
Expand All @@ -227,7 +227,7 @@ private void configureDomainRouterNicsAndDetails(VirtualMachineProfile vm, Virtu
long networkId = publicNicProfile.getNetworkId();
NetworkVO network = networkDao.findById(networkId);

for (; i < nics.length + vmwareMgr.getRouterExtraPublicNics(); i++) {
for (; i < nics.length; i++) {
NicTO nicTo = new NicTO();

nicTo.setDeviceId(deviceId++);
Expand Down
2 changes: 0 additions & 2 deletions ...s/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,6 @@ public interface VmwareManager {

Pair<Integer, Integer> getAddiionalVncPortRange();

int getRouterExtraPublicNics();

boolean beginExclusiveOperation(int timeOutSeconds);

void endExclusiveOperation();
Expand Down
8 changes: 0 additions & 8 deletions ...pervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,6 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
private String _recycleHungWorker = "false";
private int _additionalPortRangeStart;
private int _additionalPortRangeSize;
private int _routerExtraPublicNics = 2;
private int _vCenterSessionTimeout = 1200000; // Timeout in milliseconds
private String _rootDiskController = DiskControllerType.ide.toString();

Expand Down Expand Up @@ -374,8 +373,6 @@ public boolean configure(String name, Map<String, Object> params) throws Configu
_additionalPortRangeSize = Math.min(1000, 65535 - _additionalPortRangeStart);
}

_routerExtraPublicNics = NumbersUtil.parseInt(_configDao.getValue(Config.RouterExtraPublicNics.key()), 2);

_vCenterSessionTimeout = NumbersUtil.parseInt(_configDao.getValue(Config.VmwareVcenterSessionTimeout.key()), 1200) * 1000;
s_logger.info("VmwareManagerImpl config - vmware.vcenter.session.timeout: " + _vCenterSessionTimeout);

Expand Down Expand Up @@ -1069,11 +1066,6 @@ public Pair<Integer, Integer> getAddiionalVncPortRange() {
return new Pair<Integer, Integer>(_additionalPortRangeStart, _additionalPortRangeSize);
}

@Override
public int getRouterExtraPublicNics() {
return _routerExtraPublicNics;
}

@Override
public Map<String, String> getNexusVSMCredentialsByClusterId(Long clusterId) {
CiscoNexusVSMDeviceVO nexusVSM = null;
Expand Down
159 changes: 67 additions & 92 deletions ...hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1268,26 +1268,38 @@ private PlugNicAnswer execute(PlugNicCommand cmd) {
s_logger.info("Executing resource PlugNicCommand " + _gson.toJson(cmd));
}

try {
VirtualEthernetCardType nicDeviceType = null;
if (cmd.getDetails() != null) {
nicDeviceType = VirtualEthernetCardType.valueOf(cmd.getDetails().get("nicAdapter"));
}
plugNicCommandInternal(cmd.getVmName(), nicDeviceType, cmd.getNic(), cmd.getVMType());
return new PlugNicAnswer(cmd, true, "success");
} catch (Exception e) {
s_logger.error("Unexpected exception: ", e);
return new PlugNicAnswer(cmd, false, "Unable to execute PlugNicCommand due to " + e.toString());
}
}

private void plugNicCommandInternal(String vmName, VirtualEthernetCardType nicDeviceType, NicTO nicTo, VirtualMachine.Type vmType) throws Exception {
getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
VmwareContext context = getServiceContext();
try {
VmwareHypervisorHost hyperHost = getHyperHost(context);
VmwareHypervisorHost hyperHost = getHyperHost(context);

String vmName = cmd.getVmName();
VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName);
VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName);

if (vmMo == null) {
if (hyperHost instanceof HostMO) {
ClusterMO clusterMo = new ClusterMO(hyperHost.getContext(), ((HostMO) hyperHost).getParentMor());
vmMo = clusterMo.findVmOnHyperHost(vmName);
}
if (vmMo == null) {
if (hyperHost instanceof HostMO) {
ClusterMO clusterMo = new ClusterMO(hyperHost.getContext(), ((HostMO) hyperHost).getParentMor());
vmMo = clusterMo.findVmOnHyperHost(vmName);
}
}

if (vmMo == null) {
String msg = "Router " + vmName + " no longer exists to execute PlugNic command";
s_logger.error(msg);
throw new Exception(msg);
}
if (vmMo == null) {
String msg = "Router " + vmName + " no longer exists to execute PlugNic command";
s_logger.error(msg);
throw new Exception(msg);
}

/*
if(!isVMWareToolsInstalled(vmMo)){
Expand All @@ -1296,54 +1308,45 @@ private PlugNicAnswer execute(PlugNicCommand cmd) {
return new PlugNicAnswer(cmd, false, "Unable to execute PlugNicCommand due to " + errMsg);
}
*/
// Fallback to E1000 if no specific nicAdapter is passed
VirtualEthernetCardType nicDeviceType = VirtualEthernetCardType.E1000;
Map<String, String> details = cmd.getDetails();
if (details != null) {
nicDeviceType = VirtualEthernetCardType.valueOf((String) details.get("nicAdapter"));
}

// find a usable device number in VMware environment
VirtualDevice[] nicDevices = vmMo.getSortedNicDevices();
int deviceNumber = -1;
for (VirtualDevice device : nicDevices) {
if (device.getUnitNumber() > deviceNumber)
deviceNumber = device.getUnitNumber();
}
deviceNumber++;
// Fallback to E1000 if no specific nicAdapter is passed
if (nicDeviceType == null) {
nicDeviceType = VirtualEthernetCardType.E1000;
}

NicTO nicTo = cmd.getNic();
VirtualDevice nic;
Pair<ManagedObjectReference, String> networkInfo = prepareNetworkFromNicInfo(vmMo.getRunningHost(), nicTo, false, cmd.getVMType());
String dvSwitchUuid = null;
if (VmwareHelper.isDvPortGroup(networkInfo.first())) {
ManagedObjectReference dcMor = hyperHost.getHyperHostDatacenter();
DatacenterMO dataCenterMo = new DatacenterMO(context, dcMor);
ManagedObjectReference dvsMor = dataCenterMo.getDvSwitchMor(networkInfo.first());
dvSwitchUuid = dataCenterMo.getDvSwitchUuid(dvsMor);
s_logger.info("Preparing NIC device on dvSwitch : " + dvSwitchUuid);
nic = VmwareHelper.prepareDvNicDevice(vmMo, networkInfo.first(), nicDeviceType, networkInfo.second(), dvSwitchUuid,
nicTo.getMac(), deviceNumber + 1, true, true);
} else {
s_logger.info("Preparing NIC device on network " + networkInfo.second());
nic = VmwareHelper.prepareNicDevice(vmMo, networkInfo.first(), nicDeviceType, networkInfo.second(),
nicTo.getMac(), deviceNumber + 1, true, true);
}
// find a usable device number in VMware environment
VirtualDevice[] nicDevices = vmMo.getSortedNicDevices();
int deviceNumber = -1;
for (VirtualDevice device : nicDevices) {
if (device.getUnitNumber() > deviceNumber)
deviceNumber = device.getUnitNumber();
}
deviceNumber++;

VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec();
VirtualDeviceConfigSpec deviceConfigSpec = new VirtualDeviceConfigSpec();
deviceConfigSpec.setDevice(nic);
deviceConfigSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD);
VirtualDevice nic;
Pair<ManagedObjectReference, String> networkInfo = prepareNetworkFromNicInfo(vmMo.getRunningHost(), nicTo, false, vmType);
String dvSwitchUuid = null;
if (VmwareHelper.isDvPortGroup(networkInfo.first())) {
ManagedObjectReference dcMor = hyperHost.getHyperHostDatacenter();
DatacenterMO dataCenterMo = new DatacenterMO(context, dcMor);
ManagedObjectReference dvsMor = dataCenterMo.getDvSwitchMor(networkInfo.first());
dvSwitchUuid = dataCenterMo.getDvSwitchUuid(dvsMor);
s_logger.info("Preparing NIC device on dvSwitch : " + dvSwitchUuid);
nic = VmwareHelper.prepareDvNicDevice(vmMo, networkInfo.first(), nicDeviceType, networkInfo.second(), dvSwitchUuid,
nicTo.getMac(), deviceNumber + 1, true, true);
} else {
s_logger.info("Preparing NIC device on network " + networkInfo.second());
nic = VmwareHelper.prepareNicDevice(vmMo, networkInfo.first(), nicDeviceType, networkInfo.second(),
nicTo.getMac(), deviceNumber + 1, true, true);
}

vmConfigSpec.getDeviceChange().add(deviceConfigSpec);
if (!vmMo.configureVm(vmConfigSpec)) {
throw new Exception("Failed to configure devices when running PlugNicCommand");
}
VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec();
VirtualDeviceConfigSpec deviceConfigSpec = new VirtualDeviceConfigSpec();
deviceConfigSpec.setDevice(nic);
deviceConfigSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD);

return new PlugNicAnswer(cmd, true, "success");
} catch (Exception e) {
s_logger.error("Unexpected exception: ", e);
return new PlugNicAnswer(cmd, false, "Unable to execute PlugNicCommand due to " + e.toString());
vmConfigSpec.getDeviceChange().add(deviceConfigSpec);
if (!vmMo.configureVm(vmConfigSpec)) {
throw new Exception("Failed to configure devices when running PlugNicCommand");
}
}

Expand Down Expand Up @@ -1613,7 +1616,12 @@ private ExecutionResult prepareNetworkElementCommand(IpAssocCommand cmd) {
}

if (addVif) {
plugPublicNic(vmMo, vlanId, ip);
NicTO nicTO = ip.getNicTO();
VirtualEthernetCardType nicDeviceType = null;
if (ip.getDetails() != null) {
nicDeviceType = VirtualEthernetCardType.valueOf(ip.getDetails().get("nicAdapter"));
}
plugNicCommandInternal(routerName, nicDeviceType, nicTO, VirtualMachine.Type.DomainRouter);
publicNicInfo = vmMo.getNicDeviceIndex(publicNeworkName);
if (publicNicInfo.first().intValue() >= 0) {
networkUsage(controlIp, "addVif", "eth" + publicNicInfo.first());
Expand Down Expand Up @@ -2313,39 +2321,6 @@ protected StartAnswer execute(StartCommand cmd) {
int nicMask = 0;
int nicCount = 0;

if (vmSpec.getType() == VirtualMachine.Type.DomainRouter) {
int extraPublicNics = mgr.getRouterExtraPublicNics();
if (extraPublicNics > 0 && vmSpec.getDetails().containsKey("PeerRouterInstanceName")) {
//Set identical MAC address for RvR on extra public interfaces
String peerRouterInstanceName = vmSpec.getDetails().get("PeerRouterInstanceName");

VirtualMachineMO peerVmMo = hyperHost.findVmOnHyperHost(peerRouterInstanceName);
if (peerVmMo == null) {
peerVmMo = hyperHost.findVmOnPeerHyperHost(peerRouterInstanceName);
}

if (peerVmMo != null) {
String oldMacSequence = generateMacSequence(nics);

for (int nicIndex = nics.length - extraPublicNics; nicIndex < nics.length; nicIndex++) {
VirtualDevice nicDevice = peerVmMo.getNicDeviceByIndex(nics[nicIndex].getDeviceId());
if (nicDevice != null) {
String mac = ((VirtualEthernetCard) nicDevice).getMacAddress();
if (mac != null) {
s_logger.info("Use same MAC as previous RvR, the MAC is " + mac + " for extra NIC with device id: " + nics[nicIndex].getDeviceId());
nics[nicIndex].setMac(mac);
}
}
}

if (!StringUtils.isBlank(vmSpec.getBootArgs())) {
String newMacSequence = generateMacSequence(nics);
vmSpec.setBootArgs(replaceNicsMacSequenceInBootArgs(oldMacSequence, newMacSequence, vmSpec));
}
}
}
}

VirtualEthernetCardType nicDeviceType;

NiciraNvpApiVersion.logNiciraApiVersion();
Expand Down
58 changes: 58 additions & 0 deletions server/src/main/java/com/cloud/network/router/CommandSetupHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,14 @@

import javax.inject.Inject;

import com.cloud.configuration.ConfigurationManager;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.network.dao.NetworkDetailVO;
import com.cloud.network.dao.NetworkDetailsDao;
import com.cloud.offerings.dao.NetworkOfferingDetailsDao;
import com.cloud.vm.VmDetailConstants;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
Expand Down Expand Up @@ -179,6 +187,12 @@ public class CommandSetupHelper {
private RouterControlHelper _routerControlHelper;
@Inject
private HostDao _hostDao;
@Inject
ConfigurationManager _configMgr;
@Inject
private NetworkOfferingDetailsDao networkOfferingDetailsDao;
@Inject
private NetworkDetailsDao networkDetailsDao;

@Autowired
@Qualifier("networkHelper")
Expand Down Expand Up @@ -848,6 +862,15 @@ public int compare(final PublicIpAddress o1, final PublicIpAddress o2) {
vifMacAddress, networkRate, ipAddr.isOneToOneNat());

setIpAddressNetworkParams(ip, network, router);
if (router.getHypervisorType() == Hypervisor.HypervisorType.VMware) {
Map<String, String> details = new HashMap<>();
String defaultSystemVmNicAdapterType = _configDao.getValue(Config.VmwareSystemVmNicDeviceType.key());
if (defaultSystemVmNicAdapterType == null) {
defaultSystemVmNicAdapterType = Config.VmwareSystemVmNicDeviceType.getDefaultValue();
}
details.put(VmDetailConstants.NIC_ADAPTER, defaultSystemVmNicAdapterType);
ip.setDetails(details);
}
ipsToSend[i++] = ip;
/*
* send the firstIP = true for the first Add, this is to create
Expand Down Expand Up @@ -1152,6 +1175,41 @@ private void setIpAddressNetworkParams(IpAddressTO ipAddress, final Network netw
ipAddress.setPrivateGateway(false);
}
ipAddress.setNetworkName(_networkModel.getNetworkTag(router.getHypervisorType(), network));

final NetworkOfferingVO networkOfferingVO = _networkOfferingDao.findById(network.getNetworkOfferingId());
NicTO nicTO = new NicTO();
nicTO.setMac(ipAddress.getVifMacAddress());
nicTO.setType(ipAddress.getTrafficType());
nicTO.setGateway(ipAddress.getVlanGateway());
nicTO.setBroadcastUri(BroadcastDomainType.fromString(ipAddress.getBroadcastUri()));
nicTO.setType(network.getTrafficType());
nicTO.setName(_networkModel.getNetworkTag(router.getHypervisorType(), network));
nicTO.setBroadcastType(network.getBroadcastDomainType());
nicTO.setIsolationuri(network.getBroadcastUri());
nicTO.setNetworkRateMbps(_configMgr.getNetworkOfferingNetworkRate(networkOfferingVO.getId(), network.getDataCenterId()));
nicTO.setSecurityGroupEnabled(_networkModel.isSecurityGroupSupportedInNetwork(network));
nicTO.setDetails(getNicDetails(network));

ipAddress.setNicTO(nicTO);
}

private Map<NetworkOffering.Detail, String> getNicDetails(Network network) {
if (network == null) {
s_logger.debug("Unable to get NIC details as the network is null");
return null;
}
Map<NetworkOffering.Detail, String> details = networkOfferingDetailsDao.getNtwkOffDetails(network.getNetworkOfferingId());
if (details != null) {
details.putIfAbsent(NetworkOffering.Detail.PromiscuousMode, NetworkOrchestrationService.PromiscuousMode.value().toString());
details.putIfAbsent(NetworkOffering.Detail.MacAddressChanges, NetworkOrchestrationService.MacAddressChanges.value().toString());
details.putIfAbsent(NetworkOffering.Detail.ForgedTransmits, NetworkOrchestrationService.ForgedTransmits.value().toString());
details.putIfAbsent(NetworkOffering.Detail.MacLearning, NetworkOrchestrationService.MacLearning.value().toString());
}
NetworkDetailVO pvlantypeDetail = networkDetailsDao.findDetail(network.getId(), ApiConstants.ISOLATED_PVLAN_TYPE);
if (pvlantypeDetail != null) {
details.putIfAbsent(NetworkOffering.Detail.pvlanType, pvlantypeDetail.getValue());
}
return details;
}

}
2 changes: 1 addition & 1 deletion server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2063,7 +2063,7 @@ public boolean finalizeVirtualMachineProfile(final VirtualMachineProfile profile
buf.append(" dnssearchorder=").append(domain_suffix);
}

if (profile.getHypervisorType() == HypervisorType.VMware || profile.getHypervisorType() == HypervisorType.Hyperv) {
if (profile.getHypervisorType() == HypervisorType.Hyperv) {
buf.append(" extra_pubnics=" + _routerExtraPublicNics);
}

Expand Down