Switch all OSSpinLock uses to pthread_mutex_t

include/xhyve/support/misc.h

@@ -1,8 +1,11 @@
 #pragma once
 
+#include <assert.h>
+#include <pthread.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 
 #define UNUSED __attribute__ ((unused))
 #define CTASSERT(x) _Static_assert ((x), "CTASSERT")
@@ -65,3 +68,27 @@ static inline void do_cpuid(unsigned ax, unsigned *p) {
 
 /* Used to trigger a self-shutdown */
 extern void push_power_button(void);
+
+/* Error checking pthread mutex operations */
+static inline void xpthread_mutex_init(pthread_mutex_t *mutex)
+{
+	int rc = pthread_mutex_init(mutex, NULL);
+	if (__builtin_expect(rc != 0, 0))
+		xhyve_abort("pthread_mutex_init failed: %d: %s\n",
+			    rc, strerror(rc));
+}
+
+static inline void xpthread_mutex_lock(pthread_mutex_t *mutex)
+{
+	int rc = pthread_mutex_lock(mutex);
+	if (__builtin_expect(rc != 0, 0))
+		xhyve_abort("pthread_mutex_lock failed: %d: %s\n",
+			    rc, strerror(rc));
+}
+static inline void xpthread_mutex_unlock(pthread_mutex_t *mutex)
+{
+	int rc = pthread_mutex_unlock(mutex);
+	if (__builtin_expect(rc != 0, 0))
+		xhyve_abort("pthread_mutex_unlock failed: %d: %s\n",
+			    rc, strerror(rc));
+}

include/xhyve/vmm/io/vlapic_priv.h

@@ -30,7 +30,6 @@
 
 #include <stdint.h>
 #include <stdbool.h>
-#include <libkern/OSAtomic.h>
 #include <xhyve/support/apicreg.h>
 #include <xhyve/vmm/vmm_callout.h>
 
@@ -163,7 +162,8 @@ struct vlapic {
 	struct bintime timer_fire_bt; /* callout expiry time */
 	struct bintime timer_freq_bt; /* timer frequency */
 	struct bintime timer_period_bt; /* timer period */
-	OSSpinLock timer_lock;
+	pthread_mutex_t timer_lock;
+
 	/*
 	 * The 'isrvec_stk' is a stack of vectors injected by the local apic.
 	 * A vector is popped from the stack when the processor does an EOI.

src/vmm/io/vatpic.c

@@ -29,7 +29,6 @@
 #include <stdbool.h>
 #include <errno.h>
 #include <assert.h>
-#include <libkern/OSAtomic.h>
 #include <xhyve/support/misc.h>
 #include <xhyve/support/i8259.h>
 #include <xhyve/support/apicreg.h>
@@ -38,9 +37,9 @@
 #include <xhyve/vmm/io/vatpic.h>
 #include <xhyve/vmm/io/vioapic.h>
 
-#define VATPIC_LOCK_INIT(v) (v)->lock = OS_SPINLOCK_INIT;
-#define VATPIC_LOCK(v) OSSpinLockLock(&(v)->lock)
-#define VATPIC_UNLOCK(v) OSSpinLockUnlock(&(v)->lock)
+#define VATPIC_LOCK_INIT(v) xpthread_mutex_init(&(v)->lock)
+#define VATPIC_LOCK(v) xpthread_mutex_lock(&(v)->lock)
+#define VATPIC_UNLOCK(v) xpthread_mutex_unlock(&(v)->lock)
 
 enum irqstate {
 	IRQSTATE_ASSERT,
@@ -70,7 +69,7 @@ struct atpic {
 
 struct vatpic {
 	struct vm *vm;
-	OSSpinLock lock;
+	pthread_mutex_t lock;
 	struct atpic atpic[2];
 	uint8_t elc[2];
 };

src/vmm/io/vatpit.c

@@ -29,17 +29,17 @@
 #include <stdint.h>
 #include <stdbool.h>
 #include <assert.h>
-#include <libkern/OSAtomic.h>
+#include <xhyve/support/misc.h>
 #include <xhyve/support/timerreg.h>
 #include <xhyve/vmm/vmm_callout.h>
 #include <xhyve/vmm/vmm_ktr.h>
 #include <xhyve/vmm/io/vatpic.h>
 #include <xhyve/vmm/io/vatpit.h>
 #include <xhyve/vmm/io/vioapic.h>
 
-#define VATPIT_LOCK_INIT(v) (v)->lock = OS_SPINLOCK_INIT;
-#define VATPIT_LOCK(v) OSSpinLockLock(&(v)->lock)
-#define VATPIT_UNLOCK(v) OSSpinLockUnlock(&(v)->lock)
+#define VATPIT_LOCK_INIT(v) xpthread_mutex_init(&(v)->lock)
+#define VATPIT_LOCK(v) xpthread_mutex_lock(&(v)->lock)
+#define VATPIT_UNLOCK(v) xpthread_mutex_unlock(&(v)->lock)
 
 #define	TIMER_SEL_MASK		0xc0
 #define	TIMER_RW_MASK		0x30
@@ -85,7 +85,7 @@ struct channel {
 
 struct vatpit {
 	struct vm *vm;
-	OSSpinLock lock;
+	pthread_mutex_t lock;
 	sbintime_t freq_sbt;
 	struct channel channel[3];
 };
@@ -423,7 +423,7 @@ vatpit_init(struct vm *vm)
 	bzero(vatpit, sizeof(struct vatpit));
 	vatpit->vm = vm;
 
-	VATPIT_LOCK_INIT(vatpit)
+	VATPIT_LOCK_INIT(vatpit);
 
 	FREQ2BT(PIT_8254_FREQ, &bt);
 	vatpit->freq_sbt = bttosbt(bt);

src/vmm/io/vioapic.c

@@ -32,7 +32,6 @@
 #include <stdbool.h>
 #include <errno.h>
 #include <assert.h>
-#include <libkern/OSAtomic.h>
 #include <xhyve/support/misc.h>
 #include <xhyve/support/apicreg.h>
 #include <xhyve/vmm/vmm_ktr.h>
@@ -49,7 +48,7 @@
 #pragma clang diagnostic ignored "-Wpadded"
 struct vioapic {
 	struct vm *vm;
-	OSSpinLock lock;
+	pthread_mutex_t lock;
 	uint32_t id;
 	uint32_t ioregsel;
 	struct {
@@ -59,9 +58,9 @@ struct vioapic {
 };
 #pragma clang diagnostic pop
 
-#define VIOAPIC_LOCK_INIT(v) (v)->lock = OS_SPINLOCK_INIT;
-#define VIOAPIC_LOCK(v) OSSpinLockLock(&(v)->lock)
-#define VIOAPIC_UNLOCK(v) OSSpinLockUnlock(&(v)->lock)
+#define VIOAPIC_LOCK_INIT(v) xpthread_mutex_init(&(v)->lock);
+#define VIOAPIC_LOCK(v) xpthread_mutex_lock(&(v)->lock)
+#define VIOAPIC_UNLOCK(v) xpthread_mutex_unlock(&(v)->lock)
 
 #define	VIOAPIC_CTR1(vioapic, fmt, a1) \
 	VM_CTR1((vioapic)->vm, fmt, a1)

src/vmm/io/vlapic.c

@@ -31,6 +31,7 @@
 #include <stdbool.h>
 #include <strings.h>
 #include <errno.h>
+#include <assert.h>
 #include <xhyve/support/misc.h>
 #include <xhyve/support/atomic.h>
 #include <xhyve/support/specialreg.h>
@@ -56,9 +57,9 @@
  * - timer_freq_bt, timer_period_bt, timer_fire_bt
  * - timer LVT register
  */
-#define VLAPIC_TIMER_LOCK_INIT(v) (v)->timer_lock = OS_SPINLOCK_INIT;
-#define VLAPIC_TIMER_LOCK(v) OSSpinLockLock(&(v)->timer_lock)
-#define VLAPIC_TIMER_UNLOCK(v) OSSpinLockUnlock(&(v)->timer_lock)
+#define VLAPIC_TIMER_LOCK_INIT(v) xpthread_mutex_init(&(v)->timer_lock)
+#define VLAPIC_TIMER_LOCK(v) xpthread_mutex_lock(&(v)->timer_lock)
+#define VLAPIC_TIMER_UNLOCK(v) xpthread_mutex_unlock(&(v)->timer_lock)
 
 /*
  * APIC timer frequency:

src/vmm/vmm.c

@@ -69,7 +69,7 @@ struct vlapic;
  * (x) initialized before use
  */
 struct vcpu {
-	OSSpinLock lock; /* (o) protects 'state' */
+	pthread_mutex_t lock; /* (o) protects 'state' */
 	pthread_mutex_t state_sleep_mtx;
 	pthread_cond_t state_sleep_cnd;
 	pthread_mutex_t vcpu_sleep_mtx;
@@ -90,9 +90,9 @@ struct vcpu {
 	uint64_t nextrip; /* (x) next instruction to execute */
 };
 
-#define vcpu_lock_init(v) (v)->lock = OS_SPINLOCK_INIT;
-#define vcpu_lock(v) OSSpinLockLock(&(v)->lock)
-#define vcpu_unlock(v) OSSpinLockUnlock(&(v)->lock)
+#define vcpu_lock_init(v) xpthread_mutex_init(&(v)->lock)
+#define vcpu_lock(v) xpthread_mutex_lock(&(v)->lock)
+#define vcpu_unlock(v) xpthread_mutex_unlock(&(v)->lock)
 
 struct mem_seg {
 	uint64_t gpa;