Fix libvirt domain event listener by properly processing events

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -85,7 +85,6 @@
 import org.libvirt.DomainInfo.DomainState;
 import org.libvirt.DomainInterfaceStats;
 import org.libvirt.DomainSnapshot;
-import org.libvirt.Library;
 import org.libvirt.LibvirtException;
 import org.libvirt.MemoryStatistic;
 import org.libvirt.Network;
@@ -3694,20 +3693,7 @@ private StartupStorageCommand createLocalStoragePool(String localStoragePath, St
     }
 
     private void setupLibvirtEventListener() {
-        final Thread libvirtListenerThread = new Thread(() -> {
-            try {
-                Library.runEventLoop();
-            } catch (LibvirtException e) {
-                s_logger.error("LibvirtException was thrown in event loop: ", e);
-            } catch (InterruptedException e) {
-                s_logger.error("Libvirt event loop was interrupted: ", e);
-            }
-        });
-
         try {
-            libvirtListenerThread.setDaemon(true);
-            libvirtListenerThread.start();
-
             Connect conn = LibvirtConnection.getConnection();
             conn.addLifecycleListener(this::onDomainLifecycleChange);
 
@@ -3727,7 +3713,7 @@ private int onDomainLifecycleChange(Domain domain, DomainEvent domainEvent) {
                          * Checking for this helps us differentiate between events where cloudstack or admin stopped the VM vs guest
                          * initiated, and avoid pushing extra updates for actions we are initiating without a need for extra tracking */
                         DomainEventDetail detail = domainEvent.getDetail();
-                        if (StoppedDetail.SHUTDOWN.equals(detail) || StoppedDetail.CRASHED.equals(detail)) {
+                        if (StoppedDetail.SHUTDOWN.equals(detail) || StoppedDetail.CRASHED.equals(detail) || StoppedDetail.FAILED.equals(detail)) {
                             s_logger.info("Triggering out of band status update due to completed self-shutdown or crash of VM");
                             _agentStatusUpdater.triggerUpdate();
                         } else {

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java

@@ -23,6 +23,7 @@
 import com.cloud.agent.properties.AgentPropertiesFileHandler;
 import org.apache.log4j.Logger;
 import org.libvirt.Connect;
+import org.libvirt.Library;
 import org.libvirt.LibvirtException;
 
 import com.cloud.hypervisor.Hypervisor;
@@ -34,6 +35,7 @@ public class LibvirtConnection {
 
     static private Connect s_connection;
     static private String s_hypervisorURI;
+    static private Thread libvirtEventThread;
 
     static public Connect getConnection() throws LibvirtException {
         return getConnection(s_hypervisorURI);
@@ -45,6 +47,8 @@ static public Connect getConnection(String hypervisorURI) throws LibvirtExceptio
 
         if (conn == null) {
             s_logger.info("No existing libvirtd connection found. Opening a new one");
+
+            setupEventListener();
             conn = new Connect(hypervisorURI, false);
             s_logger.debug("Successfully connected to libvirt at: " + hypervisorURI);
             s_connections.put(hypervisorURI, conn);
@@ -53,7 +57,15 @@ static public Connect getConnection(String hypervisorURI) throws LibvirtExceptio
                 conn.getVersion();
             } catch (LibvirtException e) {
                 s_logger.error("Connection with libvirtd is broken: " + e.getMessage());
+
+                try {
+                    conn.close();
+                } catch (LibvirtException closeEx) {
+                    s_logger.debug("Ignoring error while trying to close broken connection:" + closeEx.getMessage());
+                }
+
                 s_logger.debug("Opening a new libvirtd connection to: " + hypervisorURI);
+                setupEventListener();
                 conn = new Connect(hypervisorURI, false);
                 s_connections.put(hypervisorURI, conn);
             }
@@ -101,4 +113,33 @@ static String getHypervisorURI(String hypervisorType) {
 
         return "qemu:///system";
     }
+
+    /**
+     * Set up Libvirt event handling and polling. This is not specific to a connection object instance, but needs
+     * to be done prior to creating connections. See the Libvirt documentation for virEventRegisterDefaultImpl and
+     * virEventRunDefaultImpl or the libvirt-java Library Javadoc for more information.
+     * @throws LibvirtException
+     */
+    private static synchronized void setupEventListener() throws LibvirtException {
+        if (libvirtEventThread == null || !libvirtEventThread.isAlive()) {
+            // Registers a default event loop, must be called before connecting to hypervisor
+            Library.initEventLoop();
+            libvirtEventThread = new Thread(() -> {
+                while (true) {
+                    try {
+                        // This blocking call contains a loop of its own that will process events until the event loop is stopped or exception is thrown.
+                        Library.runEventLoop();
+                    } catch (LibvirtException e) {
+                        s_logger.error("LibvirtException was thrown in event loop: ", e);
+                    } catch (InterruptedException e) {
+                        s_logger.error("Libvirt event loop was interrupted: ", e);
+                    }
+                }
+            });
+
+            // Process events in separate thread. Failure to run event loop regularly will cause connections to close due to keepalive timeout.
+            libvirtEventThread.setDaemon(true);
+            libvirtEventThread.start();
+        }
+    }
 }