Add global settings configurations; enhance documentation; work on tests

Change VMs counting
Fix test case issues and enhance documentations and code

plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java

@@ -26,13 +26,12 @@
 import com.cloud.host.Status;
 import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor;
-import com.cloud.hypervisor.kvm.resource.KvmAgentHaClient;
 import com.cloud.resource.ResourceManager;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.utils.component.AdapterBase;
-import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.dao.VMInstanceDao;
 import org.apache.cloudstack.ha.HAManager;
+import org.apache.cloudstack.kvm.ha.KvmHaAgentClient;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.log4j.Logger;
@@ -86,21 +85,29 @@ public Status isAgentAlive(Host agent) {
         boolean hasNfs = isHostServedByNfsPool(agent);
         if (hasNfs) {
             agentStatus = checkAgentStatusViaNfs(agent);
-            s_logger.debug(String.format("Agent investigation was requested on host %s, agent status via NFS storage is %s.", agent, agentStatus));
+            s_logger.debug(String.format("Agent investigation was requested on host %s. Agent status via NFS heartbeat is %s.", agent, agentStatus));
         } else {
             s_logger.debug(String.format("Agent investigation was requested on host %s, but host has no NFS storage. Skipping investigation via NFS.", agent));
         }
 
-        List<VMInstanceVO> vmsOnHost = vmInstanceDao.listByHostId(agent.getId());
-        KvmAgentHaClient kvmAgentHaClient = new KvmAgentHaClient(agent);
-        boolean isVmsOnKvmMatchingWithDatabase = kvmAgentHaClient.checkAgentHealthAndRunningVms(vmsOnHost.size());
-        if(isVmsOnKvmMatchingWithDatabase) {
+        agentStatus = checkAgentStatusViaKvmHaAgent(agent, agentStatus);
+
+        return agentStatus;
+    }
+
+    /**
+     * It checks the KVM node healthy via KVM HA Agent. If the agent is healthy it returns Status.Up, otherwise it relies keeps the provided Status as it is.
+     */
+    private Status checkAgentStatusViaKvmHaAgent(Host agent, Status agentStatus) {
+        KvmHaAgentClient kvmHaAgentClient = new KvmHaAgentClient(agent);
+
+        boolean isVmsCountOnKvmMatchingWithDatabase = kvmHaAgentClient.isKvmHaAgentHealthy(agent, vmInstanceDao);
+        if(isVmsCountOnKvmMatchingWithDatabase) {
             agentStatus = Status.Up;
-            s_logger.debug(String.format("Checking agent %s status; KVM HA webserver is Running as expected."));
+            s_logger.debug(String.format("Checking agent %s status; KVM HA Agent is Running as expected."));
         } else {
-            s_logger.warn(String.format("Checking agent %s status. Failed to check host status via KVM Agent HA webserver"));
+            s_logger.warn(String.format("Checking agent %s status. Failed to check host status via KVM HA Agent"));
         }
-
         return agentStatus;
     }
 

plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java

@@ -53,4 +53,11 @@ public class KVMHAConfig {
     public static final ConfigKey<Long> KvmHAFenceTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.fence.timeout", "60",
             "The maximum length of time, in seconds, expected for a fence operation to complete.", true, ConfigKey.Scope.Cluster);
 
+    public static final ConfigKey<Integer> KVM_HA_WEBSERVICE_PORT = new ConfigKey<Integer>("Advanced", Integer.class, "kvm.ha.webservice.port", "8080",
+            "It sets the port used to communicate with the KVM HA Agent Microservice that is running on KVM nodes. Default value is 8080.",
+            true, ConfigKey.Scope.Cluster);
+
+    public static final ConfigKey<Boolean> KVM_HA_WEBSERVICE_ENABLED = new ConfigKey<Boolean>("Advanced", Boolean.class, "kvm.ha.webservice.enabled", "true",
+            "The KVM HA Webservice is executed on the KVM node and checks the amount of VMs running via libvirt. It serves as a HA health-check for KVM nodes. One can enable (set to 'true') or disable it ('false'). If disabled then CloudStack ignores HA validation via this agent.",
+            true, ConfigKey.Scope.Cluster);
 }

plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java

@@ -151,7 +151,9 @@ public ConfigKey<?>[] getConfigKeys() {
             KVMHAConfig.KvmHAActivityCheckFailureThreshold,
             KVMHAConfig.KvmHADegradedMaxPeriod,
             KVMHAConfig.KvmHARecoverWaitPeriod,
-            KVMHAConfig.KvmHARecoverAttemptThreshold
+            KVMHAConfig.KvmHARecoverAttemptThreshold,
+            KVMHAConfig.KVM_HA_WEBSERVICE_PORT,
+            KVMHAConfig.KVM_HA_WEBSERVICE_ENABLED
         };
     }
 }

plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java

@@ -21,12 +21,13 @@
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.CheckOnHostCommand;
 import com.cloud.agent.api.CheckVMActivityOnStoragePoolCommand;
+import com.cloud.dc.ClusterVO;
+import com.cloud.dc.dao.ClusterDao;
 import com.cloud.exception.StorageUnavailableException;
 import com.cloud.host.Host;
 import com.cloud.host.HostVO;
 import com.cloud.host.Status;
 import com.cloud.hypervisor.Hypervisor;
-import com.cloud.hypervisor.kvm.resource.KvmAgentHaClient;
 import com.cloud.resource.ResourceManager;
 import com.cloud.storage.Storage;
 import com.cloud.storage.StorageManager;
@@ -51,6 +52,7 @@
 import java.util.HashMap;
 import java.util.List;
 
+
 public class KVMHostActivityChecker extends AdapterBase implements ActivityCheckerInterface<Host>, HealthCheckerInterface<Host> {
     private final static Logger LOG = Logger.getLogger(KVMHostActivityChecker.class);
 
@@ -66,6 +68,8 @@ public class KVMHostActivityChecker extends AdapterBase implements ActivityCheck
     private StorageManager storageManager;
     @Inject
     private ResourceManager resourceManager;
+    @Inject
+    private ClusterDao clusterDao;
 
     @Override
     public boolean isActive(Host r, DateTime suspectTime) throws HACheckerException {
@@ -87,21 +91,41 @@ public boolean isHealthy(Host r) {
         HashMap<StoragePool, List<Volume>> poolVolMap = getVolumeUuidOnHost(r);
         isHealthy = isHealthyCheckViaNfs(r, isHealthy, poolVolMap);
 
-        KvmAgentHaClient kvmAgentHaClient = new KvmAgentHaClient(r);
-        List<VMInstanceVO> vmsOnHost = vmInstanceDao.listByHostId(r.getId());
-        boolean checkKvmHeatlh = kvmAgentHaClient.checkAgentHealthAndRunningVms(vmsOnHost.size());
+        isHealthy = checkHealthViaKvmHaWebservice(r, isHealthy);
 
-        if(!isHealthy && checkKvmHeatlh) {
-            isHealthy = true;
+        return isHealthy;
+    }
+
+    /**
+     * Checks the host healthy via an web-service that retrieves Running KVM instances via libvirt. <br>
+     * The health-check is executed on the KVM node and verifies the amount of VMs running and if the libvirt service is running. <br><br>
+     *
+     * One can enable or disable it via global settings 'kvm.ha.webservice.enabled'.
+     */
+    private boolean checkHealthViaKvmHaWebservice(Host r, boolean isHealthy) {
+        KvmHaAgentClient kvmHaAgentClient = new KvmHaAgentClient(r);
+        if(!kvmHaAgentClient.isKvmHaWebserviceEnabled()) {
+            ClusterVO cluster = clusterDao.findById(r.getClusterId());
+            LOG.debug(String.format("Skipping KVM HA web-service verification for %s due to 'kvm.ha.webservice.enabled' not enabled for cluster [id: %d, name: %s].",
+                    r.toString(), cluster.getId(), cluster.getName()));
+            return isHealthy;
         }
 
+//        List<VMInstanceVO> vmsOnHost = kvmHaAgentClient.listVmsRunningMigratingStopping(r);
+//        List<VMInstanceVO> vmsOnHost = vmInstanceDao.listByHostAndState(r.getId(), VirtualMachine.State.Running);
+//        vmsOnHost.addAll(vmInstanceDao.listByHostAndState(r.getId(), VirtualMachine.State.Stopping));
+//        vmsOnHost.addAll(vmInstanceDao.listByHostAndState(r.getId(), VirtualMachine.State.Migrating));
+        boolean isKvmHaAgentHealthy = kvmHaAgentClient.isKvmHaAgentHealthy(r, vmInstanceDao);
+
+        if (!isHealthy && isKvmHaAgentHealthy) {
+            isHealthy = true;
+        }
         return isHealthy;
     }
 
     private boolean isHealthyCheckViaNfs(Host r, boolean isHealthy, HashMap<StoragePool, List<Volume>> poolVolMap) {
         for (StoragePool pool : poolVolMap.keySet()) {
             if(Storage.StoragePoolType.NetworkFilesystem == pool.getPoolType()
-                    || Storage.StoragePoolType.ManagedNFS == pool.getPoolType()
                     || Storage.StoragePoolType.ManagedNFS == pool.getPoolType()) {
                 isHealthy = isAgentActive(r);
             }
@@ -176,35 +200,29 @@ private boolean isVMActivtyOnHost(Host agent, DateTime suspectTime) throws HAChe
         if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM && agent.getHypervisorType() != Hypervisor.HypervisorType.LXC) {
             throw new IllegalStateException(String.format("Calling KVM investigator for non KVM Host of type [%s].", agent.getHypervisorType()));
         }
-        boolean activityStatus = false;
+        boolean activityStatus = true;
         HashMap<StoragePool, List<Volume>> poolVolMap = getVolumeUuidOnHost(agent);
         for (StoragePool pool : poolVolMap.keySet()) {
             if(Storage.StoragePoolType.NetworkFilesystem == pool.getPoolType()
-                    || Storage.StoragePoolType.ManagedNFS == pool.getPoolType()
                     || Storage.StoragePoolType.ManagedNFS == pool.getPoolType()) {
-                activityStatus = verifyActivityOfStorageOnHost(poolVolMap, pool, agent, suspectTime, activityStatus);
+                activityStatus = checkVmActivityOnStoragePool(poolVolMap, pool, agent, suspectTime, activityStatus);
                 if (!activityStatus) {
                     LOG.warn(String.format("It seems that the storage pool [%s] does not have activity on %s.", pool.getId(), agent.toString()));
                     break;
                 }
             }
         }
 
-        KvmAgentHaClient kvmAgentHaClient = new KvmAgentHaClient(agent);
-        List<VMInstanceVO> vmsOnHost = vmInstanceDao.listByHostId(agent.getId());
-        boolean isVmsOnKvmMatchingWithDatabase = kvmAgentHaClient.checkAgentHealthAndRunningVms(vmsOnHost.size());
+        activityStatus = checkHealthViaKvmHaWebservice(agent, activityStatus);
 
-        if(!activityStatus && isVmsOnKvmMatchingWithDatabase) {
-            activityStatus = true;
-        } else {
+        if(!activityStatus){
             LOG.warn(String.format("No VM activity detected on %s. This might trigger HA Host Recovery and/or Fence.", agent.toString()));
         }
 
         return activityStatus;
     }
 
-
-    protected boolean verifyActivityOfStorageOnHost(HashMap<StoragePool, List<Volume>> poolVolMap, StoragePool pool, Host agent, DateTime suspectTime, boolean activityStatus) throws HACheckerException, IllegalStateException {
+    private boolean checkVmActivityOnStoragePool(HashMap<StoragePool, List<Volume>> poolVolMap, StoragePool pool, Host agent, DateTime suspectTime, boolean activityStatus) throws HACheckerException, IllegalStateException {
         List<Volume> volume_list = poolVolMap.get(pool);
         final CheckVMActivityOnStoragePoolCommand cmd = new CheckVMActivityOnStoragePoolCommand(agent, pool, volume_list, suspectTime);