From 72280719244d0957da2fc80bf25477febb15244f Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Wed, 3 Jan 2024 11:02:04 -0700 Subject: [PATCH 1/2] Fix libvirt domain event listener by properly processing events --- .../resource/LibvirtComputingResource.java | 16 +------- .../kvm/resource/LibvirtConnection.java | 37 +++++++++++++++++++ 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 60e6bcffeb61..fef5d4aa6de1 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -85,7 +85,6 @@ import org.libvirt.DomainInfo.DomainState; import org.libvirt.DomainInterfaceStats; import org.libvirt.DomainSnapshot; -import org.libvirt.Library; import org.libvirt.LibvirtException; import org.libvirt.MemoryStatistic; import org.libvirt.Network; @@ -3694,20 +3693,7 @@ private StartupStorageCommand createLocalStoragePool(String localStoragePath, St } private void setupLibvirtEventListener() { - final Thread libvirtListenerThread = new Thread(() -> { - try { - Library.runEventLoop(); - } catch (LibvirtException e) { - s_logger.error("LibvirtException was thrown in event loop: ", e); - } catch (InterruptedException e) { - s_logger.error("Libvirt event loop was interrupted: ", e); - } - }); - try { - libvirtListenerThread.setDaemon(true); - libvirtListenerThread.start(); - Connect conn = LibvirtConnection.getConnection(); conn.addLifecycleListener(this::onDomainLifecycleChange); @@ -3727,7 +3713,7 @@ private int onDomainLifecycleChange(Domain domain, DomainEvent domainEvent) { * Checking for this helps us differentiate between events where cloudstack or admin stopped the VM vs guest * initiated, and avoid pushing extra updates for actions we are initiating without a need for extra tracking */ DomainEventDetail detail = domainEvent.getDetail(); - if (StoppedDetail.SHUTDOWN.equals(detail) || StoppedDetail.CRASHED.equals(detail)) { + if (StoppedDetail.SHUTDOWN.equals(detail) || StoppedDetail.CRASHED.equals(detail) || StoppedDetail.FAILED.equals(detail)) { s_logger.info("Triggering out of band status update due to completed self-shutdown or crash of VM"); _agentStatusUpdater.triggerUpdate(); } else { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java index 7563f9647596..e26a524134b0 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java @@ -23,6 +23,7 @@ import com.cloud.agent.properties.AgentPropertiesFileHandler; import org.apache.log4j.Logger; import org.libvirt.Connect; +import org.libvirt.Library; import org.libvirt.LibvirtException; import com.cloud.hypervisor.Hypervisor; @@ -34,6 +35,7 @@ public class LibvirtConnection { static private Connect s_connection; static private String s_hypervisorURI; + static private Thread libvirtEventThread; static public Connect getConnection() throws LibvirtException { return getConnection(s_hypervisorURI); @@ -45,6 +47,8 @@ static public Connect getConnection(String hypervisorURI) throws LibvirtExceptio if (conn == null) { s_logger.info("No existing libvirtd connection found. Opening a new one"); + + setupEventListener(); conn = new Connect(hypervisorURI, false); s_logger.debug("Successfully connected to libvirt at: " + hypervisorURI); s_connections.put(hypervisorURI, conn); @@ -53,7 +57,15 @@ static public Connect getConnection(String hypervisorURI) throws LibvirtExceptio conn.getVersion(); } catch (LibvirtException e) { s_logger.error("Connection with libvirtd is broken: " + e.getMessage()); + + try { + conn.close(); + } catch (LibvirtException closeEx) { + s_logger.debug("Ignoring error while trying to close broken connection:" + closeEx.getMessage()); + } + s_logger.debug("Opening a new libvirtd connection to: " + hypervisorURI); + setupEventListener(); conn = new Connect(hypervisorURI, false); s_connections.put(hypervisorURI, conn); } @@ -101,4 +113,29 @@ static String getHypervisorURI(String hypervisorType) { return "qemu:///system"; } + + // stand up libvirt event handling and polling. This is not specific to a connection object instance, but needs to + // exist prior to creating connections. + private static synchronized void setupEventListener() throws LibvirtException { + if (libvirtEventThread == null || !libvirtEventThread.isAlive()) { + // Registers a default event loop, must be called before connecting to hypervisor + Library.initEventLoop(); + libvirtEventThread = new Thread(() -> { + while (true) { + try { + // This blocking call contains a loop of its own that will process events until the event loop is stopped or exception is thrown. + Library.runEventLoop(); + } catch (LibvirtException e) { + s_logger.error("LibvirtException was thrown in event loop: ", e); + } catch (InterruptedException e) { + s_logger.error("Libvirt event loop was interrupted: ", e); + } + } + }); + + // Process events in separate thread. Failure to run event loop regularly will cause connections to close due to keepalive timeout. + libvirtEventThread.setDaemon(true); + libvirtEventThread.start(); + } + } } From 96eeed62b0578dc83eee79236286802cc4317524 Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Thu, 4 Jan 2024 09:58:40 -0700 Subject: [PATCH 2/2] Add javadoc for setupEventListener --- .../cloud/hypervisor/kvm/resource/LibvirtConnection.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java index e26a524134b0..1f4ab23a43fc 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java @@ -114,8 +114,12 @@ static String getHypervisorURI(String hypervisorType) { return "qemu:///system"; } - // stand up libvirt event handling and polling. This is not specific to a connection object instance, but needs to - // exist prior to creating connections. + /** + * Set up Libvirt event handling and polling. This is not specific to a connection object instance, but needs + * to be done prior to creating connections. See the Libvirt documentation for virEventRegisterDefaultImpl and + * virEventRunDefaultImpl or the libvirt-java Library Javadoc for more information. + * @throws LibvirtException + */ private static synchronized void setupEventListener() throws LibvirtException { if (libvirtEventThread == null || !libvirtEventThread.isAlive()) { // Registers a default event loop, must be called before connecting to hypervisor