Merge branch 'master' into dev

* master:
  server: add conditions for custom offerings (apache#4540)
  vr: Ensuring dnsmasq.leases file is populated (apache#4529)
  template: Ensuring template is cross zone if type changed to system (apache#4522)
  storage: Fix hypervisor type cast to string (apache#4516)
  db upgrade: fix sql exception: Access denied; you need (at least one of) the SUPER privilege(s) for this operation (apache#4533)
  CLOUDSTACK-10423:Potential sensitive information disclosure (apache#4536)
  jobs: The patch remove the password from resultObject and make it be humanreadable (apache#4538)
  listphysicalnetworks: Honouring keyword parameter (apache#4511)
  Fix NPE when Volume exists on secondary store but doesn't have a download URL (apache#4530)
  apidoc issue (apache#4532)
  db: Fix description of volume.stats.interval which is in milliseconds not seconds (apache#4526)
  kvm: set cpu topology only if cpucore per socket is positive value (apache#4527)
  xenserver: check and eject patch vbd for systemvms (apache#4525)
  Fix warning when setup cloudstack-common (apache#4523)
  kvm: FIX cpucorespersocket is not working on KVM (apache#4497)
  change debug to warn for unknown exceptions (apache#4521)
  Fix failure in validating IP address in case of multiple Management Servers (apache#4507)
  Update log output for FirstFitPlanner (apache#4515)
  ui: deprecate old UI and move to legacy to be served at /client/legacy (apache#4518)

Author: qrry

engine/schema/src/main/resources/META-INF/db/schema-410to420.sql

@@ -1288,7 +1288,7 @@ UPDATE `cloud`.`account` SET `cloud`.`account`.`default`=1 WHERE id IN (1,2);
 UPDATE `cloud_usage`.`account` SET `default`=1 WHERE id IN (1,2);
 UPDATE `cloud`.`user` SET `cloud`.`user`.`default`=1 WHERE id IN (1,2);
 
-ALTER VIEW `cloud`.`user_view` AS
+CREATE OR REPLACE VIEW `cloud`.`user_view` AS
     select 
         user.id,
         user.uuid,

engine/schema/src/main/resources/META-INF/db/schema-41120to41200.sql

@@ -60,7 +60,7 @@ alter table `cloud`.`disk_offering` add `iops_read_rate_max_length` bigint(20) d
 alter table `cloud`.`disk_offering` add `iops_write_rate_max` bigint(20) default null after `iops_write_rate`;
 alter table `cloud`.`disk_offering` add `iops_write_rate_max_length` bigint(20) default null after `iops_write_rate_max`;
 
-ALTER VIEW `cloud`.`disk_offering_view` AS
+CREATE OR REPLACE VIEW `cloud`.`disk_offering_view` AS
     SELECT
         `disk_offering`.`id` AS `id`,
         `disk_offering`.`uuid` AS `uuid`,
@@ -105,7 +105,7 @@ ALTER VIEW `cloud`.`disk_offering_view` AS
         (`disk_offering`.`state` = 'ACTIVE');
 
 
-ALTER VIEW `cloud`.`service_offering_view` AS
+CREATE OR REPLACE VIEW `cloud`.`service_offering_view` AS
     SELECT
         `service_offering`.`id` AS `id`,
         `disk_offering`.`uuid` AS `uuid`,
@@ -159,4 +159,4 @@ ALTER VIEW `cloud`.`service_offering_view` AS
         (`disk_offering`.`state` = 'Active');
 
 -- PR#2578 New column for listManagementServers API call
-ALTER TABLE `mshost` ADD COLUMN `uuid` varchar(40) AFTER `name`;
+ALTER TABLE `mshost` ADD COLUMN `uuid` varchar(40) AFTER `name`;

engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql

@@ -840,3 +840,6 @@ ALTER TABLE `cloud_usage`.`cloud_usage` ADD COLUMN `is_hidden` smallint(1) NOT N
 
 -- Fix Zones are returned in a random order (#3934)
 UPDATE `cloud`.`data_center` JOIN (SELECT COUNT(1) AS count FROM `cloud`.`data_center` WHERE `sort_key` != 0) AS tbl_tmp SET `sort_key` = `id` WHERE count = 0;
+
+-- Fix description of volume.stats.interval which is in milliseconds not seconds
+UPDATE `cloud`.`configuration` SET `description` = 'Interval (in milliseconds) to report volume statistics' WHERE `name` = 'volume.stats.interval';

engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/VolumeServiceTest.java

@@ -276,6 +276,7 @@ public void testCreatePrimaryStorage() {
         params.put("path", uri.getPath());
         params.put("protocol", StoragePoolType.NetworkFilesystem);
         params.put("dcId", dcId.toString());
+        params.put("hypervisorType", HypervisorType.None);
         params.put("clusterId", clusterId.toString());
         params.put("name", this.primaryName);
         params.put("port", "1");
@@ -318,6 +319,7 @@ public DataStore createPrimaryDataStore() {
             params.put("path", uri.getPath());
             params.put("protocol", Storage.StoragePoolType.NetworkFilesystem);
             params.put("dcId", dcId.toString());
+            params.put("hypervisorType", HypervisorType.None);
             params.put("clusterId", clusterId.toString());
             params.put("name", this.primaryName);
             params.put("port", "1");

framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java

@@ -257,9 +257,9 @@ public Long doInTransaction(TransactionStatus status) {
     @Override
     @DB
     public void completeAsyncJob(final long jobId, final Status jobStatus, final int resultCode, final String resultObject) {
+        String resultObj = null;
         if (s_logger.isDebugEnabled()) {
-            String resultObj = obfuscatePassword(resultObject, HidePassword.value());
-            resultObj = convertHumanReadableJson(resultObj);
+            resultObj = convertHumanReadableJson(obfuscatePassword(resultObject, HidePassword.value()));
             s_logger.debug("Complete async job-" + jobId + ", jobStatus: " + jobStatus + ", resultCode: " + resultCode + ", result: " + resultObj);
         }
 
@@ -268,7 +268,7 @@ public void completeAsyncJob(final long jobId, final Status jobStatus, final int
         if (job == null) {
             if (s_logger.isDebugEnabled()) {
                 s_logger.debug("job-" + jobId + " no longer exists, we just log completion info here. " + jobStatus + ", resultCode: " + resultCode + ", result: " +
-                    resultObject);
+                    resultObj);
             }
             // still purge item from queue to avoid any blocking
             _queueMgr.purgeAsyncJobQueueItemId(jobId);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -2302,14 +2302,7 @@ public LibvirtVMDef createVMFromSpec(final VirtualMachineTO vmTO) {
             if (vmTO.getType() == VirtualMachine.Type.User) {
                 cmd.setFeatures(_cpuFeatures);
             }
-            // multi cores per socket, for larger core configs
-            if (vcpus % 6 == 0) {
-                final int sockets = vcpus / 6;
-                cmd.setTopology(6, sockets);
-            } else if (vcpus % 4 == 0) {
-                final int sockets = vcpus / 4;
-                cmd.setTopology(4, sockets);
-            }
+            setCpuTopology(cmd, vcpus, vmTO.getDetails());
             vm.addComp(cmd);
         }
 
@@ -4230,4 +4223,26 @@ public boolean isSecureMode(String bootMode) {
 
         return false;
     }
+
+    private void setCpuTopology(CpuModeDef cmd, int vcpus, Map<String, String> details) {
+        // multi cores per socket, for larger core configs
+        int numCoresPerSocket = -1;
+        if (details != null) {
+            final String coresPerSocket = details.get(VmDetailConstants.CPU_CORE_PER_SOCKET);
+            final int intCoresPerSocket = NumbersUtil.parseInt(coresPerSocket, numCoresPerSocket);
+            if (intCoresPerSocket > 0 && vcpus % intCoresPerSocket == 0) {
+                numCoresPerSocket = intCoresPerSocket;
+            }
+        }
+        if (numCoresPerSocket <= 0) {
+            if (vcpus % 6 == 0) {
+                numCoresPerSocket = 6;
+            } else if (vcpus % 4 == 0) {
+                numCoresPerSocket = 4;
+            }
+        }
+        if (numCoresPerSocket > 0) {
+            cmd.setTopology(numCoresPerSocket, vcpus / numCoresPerSocket);
+        }
+    }
 }

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/discoverer/XcpServerDiscoverer.java

@@ -369,7 +369,7 @@ protected boolean poolHasHotFix(Connection conn, String hostIp, String hotFixUui
             s_logger.warn("Unable to resolve the host name", e);
             return null;
         } catch (Exception e) {
-            s_logger.debug("other exceptions: " + e.toString(), e);
+            s_logger.warn("other exceptions: " + e.toString(), e);
             return null;
         }
         return resources;

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java

@@ -1063,33 +1063,46 @@ protected SR createNfsSRbyURI(final Connection conn, final URI uri, final boolea
         }
     }
 
-    public VBD createPatchVbd(final Connection conn, final String vmName, final VM vm) throws XmlRpcException, XenAPIException {
+    public SR findPatchIsoSR(final Connection conn) throws XmlRpcException, XenAPIException {
+        Set<SR> srs = SR.getByNameLabel(conn, "XenServer Tools");
+        if (srs.size() != 1) {
+            s_logger.debug("Failed to find SR by name 'XenServer Tools', will try to find 'XCP-ng Tools' SR");
+            srs = SR.getByNameLabel(conn, "XCP-ng Tools");
+        }
+        if (srs.size() != 1) {
+            s_logger.debug("Failed to find SR by name 'XenServer Tools' or 'XCP-ng Tools', will try to find 'Citrix Hypervisor' SR");
+            srs = SR.getByNameLabel(conn, "Citrix Hypervisor Tools");
+        }
+        if (srs.size() != 1) {
+            throw new CloudRuntimeException("There are " + srs.size() + " SRs with name XenServer Tools or XCP-ng Tools or Citrix Hypervisor Tools");
+        }
+        final SR sr = srs.iterator().next();
+        sr.scan(conn);
+        return sr;
+    }
 
-        if (_host.getSystemvmisouuid() == null) {
-            Set<SR> srs = SR.getByNameLabel(conn, "XenServer Tools");
-            if (srs.size() != 1) {
-                s_logger.debug("Failed to find SR by name 'XenServer Tools', will try to find 'XCP-ng Tools' SR");
-                srs = SR.getByNameLabel(conn, "XCP-ng Tools");
-            }
-            if (srs.size() != 1) {
-                s_logger.debug("Failed to find SR by name 'XenServer Tools' or 'XCP-ng Tools', will try to find 'Citrix Hypervisor' SR");
-                srs = SR.getByNameLabel(conn, "Citrix Hypervisor Tools");
-            }
-            if (srs.size() != 1) {
-                throw new CloudRuntimeException("There are " + srs.size() + " SRs with name XenServer Tools or XCP-ng Tools or Citrix Hypervisor Tools");
+    public VDI findPatchIsoVDI(final Connection conn, final SR sr) throws XmlRpcException, XenAPIException {
+        if (sr == null) {
+            return null;
+        }
+        final SR.Record srr = sr.getRecord(conn);
+        for (final VDI vdi : srr.VDIs) {
+            final VDI.Record vdir = vdi.getRecord(conn);
+            if (vdir.nameLabel.contains("systemvm.iso")) {
+                return vdi;
             }
-            final SR sr = srs.iterator().next();
-            sr.scan(conn);
+        }
+        return null;
+    }
 
-            final SR.Record srr = sr.getRecord(conn);
+    public VBD createPatchVbd(final Connection conn, final String vmName, final VM vm) throws XmlRpcException, XenAPIException {
 
+        if (_host.getSystemvmisouuid() == null) {
+            final SR sr = findPatchIsoSR(conn);
             if (_host.getSystemvmisouuid() == null) {
-                for (final VDI vdi : srr.VDIs) {
-                    final VDI.Record vdir = vdi.getRecord(conn);
-                    if (vdir.nameLabel.contains("systemvm.iso")) {
-                        _host.setSystemvmisouuid(vdir.uuid);
-                        break;
-                    }
+                final VDI vdi = findPatchIsoVDI(conn, sr);
+                if (vdi != null) {
+                    _host.setSystemvmisouuid(vdi.getRecord(conn).uuid);
                 }
             }
             if (_host.getSystemvmisouuid() == null) {
@@ -1489,24 +1502,33 @@ protected String deleteSnapshotBackup(final Connection conn, final Long dcId, fi
         return result;
     }
 
-    public void destroyPatchVbd(final Connection conn, final String vmName) throws XmlRpcException, XenAPIException {
-        try {
-            if (!vmName.startsWith("r-") && !vmName.startsWith("s-") && !vmName.startsWith("v-")) {
-                return;
-            }
-            final Set<VM> vms = VM.getByNameLabel(conn, vmName);
-            for (final VM vm : vms) {
+    public void destroyPatchVbd(final Connection conn, final Set<VM> vms) throws XmlRpcException, XenAPIException {
+        final SR sr = findPatchIsoSR(conn);
+        final VDI patchVDI = findPatchIsoVDI(conn, sr);
+        for (final VM vm : vms) {
+            final String vmName = vm.getNameLabel(conn);
+            try {
+                if (!vmName.startsWith("r-") && !vmName.startsWith("s-") && !vmName.startsWith("v-")) {
+                    return;
+                }
                 final Set<VBD> vbds = vm.getVBDs(conn);
                 for (final VBD vbd : vbds) {
-                    if (vbd.getType(conn) == Types.VbdType.CD) {
-                        vbd.eject(conn);
+                    if (Types.VbdType.CD.equals(vbd.getType(conn))) {
+                        if (!vbd.getEmpty(conn)) {
+                            vbd.eject(conn);
+                        }
+                        // Workaround for any file descriptor caching issue
+                        if (patchVDI != null) {
+                            vbd.insert(conn, patchVDI);
+                            vbd.eject(conn);
+                        }
                         vbd.destroy(conn);
                         break;
                     }
                 }
+            } catch (final Exception e) {
+                s_logger.debug("Cannot destroy CD-ROM device for VM " + vmName + " due to " + e.toString(), e);
             }
-        } catch (final Exception e) {
-            s_logger.debug("Cannot destory CD-ROM device for VM " + vmName + " due to " + e.toString(), e);
         }
     }
 

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixReadyCommandWrapper.java

@@ -54,9 +54,7 @@ public Answer execute(final ReadyCommand command, final CitrixResourceBase citri
         try {
             final Host host = Host.getByUuid(conn, citrixResourceBase.getHost().getUuid());
             final Set<VM> vms = host.getResidentVMs(conn);
-            for (final VM vm : vms) {
-                citrixResourceBase.destroyPatchVbd(conn, vm.getNameLabel(conn));
-            }
+            citrixResourceBase.destroyPatchVbd(conn, vms);
         } catch (final Exception e) {
         }
         try {

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixSetupCommandWrapper.java

@@ -77,7 +77,6 @@ public Answer execute(final SetupCommand command, final CitrixResourceBase citri
 
             }
 
-
             final boolean r = citrixResourceBase.launchHeartBeat(conn);
             if (!r) {
                 return null;