From 6a7e353dc5cbead3051eca4520f6f3e2f888289c Mon Sep 17 00:00:00 2001 From: Melvin Hillsman Date: Fri, 22 May 2026 01:36:13 -0500 Subject: [PATCH] fix: use virtio serial for stable data disk discovery in PVC-backed workloads Replace fragile /dev/vdX device paths with serial-based discovery via /dev/disk/by-id/virtio- for chaos-disk, disk, and database workloads. Adds diskSetupScript() helper for shared format/mount logic. Fixes #58 Signed-off-by: Melvin Hillsman --- internal/workloads/chaos_disk.go | 10 +++++-- internal/workloads/chaos_disk_test.go | 27 +++++++++++++++-- internal/workloads/database.go | 25 +++++++++++++--- internal/workloads/database_test.go | 6 ++-- internal/workloads/disk.go | 10 +++++-- internal/workloads/disk_test.go | 31 +++++++++++++++++--- internal/workloads/workload.go | 42 +++++++++++++++++++++++++++ 7 files changed, 135 insertions(+), 16 deletions(-) diff --git a/internal/workloads/chaos_disk.go b/internal/workloads/chaos_disk.go index 47e4713..f1cf723 100644 --- a/internal/workloads/chaos_disk.go +++ b/internal/workloads/chaos_disk.go @@ -86,6 +86,11 @@ func (w *ChaosDiskWorkload) Name() string { func (w *ChaosDiskWorkload) CloudInitUserdata() (string, error) { return w.BuildCloudConfig(CloudConfigOpts{ WriteFiles: []WriteFile{ + { + Path: "/usr/local/bin/virtwork-disk-setup.sh", + Content: diskSetupScript("virtwork-chdisk", "/mnt/data"), + Permissions: "0755", + }, { Path: "/usr/local/bin/chaos-disk.sh", Content: chaosDiskScript, @@ -98,7 +103,7 @@ func (w *ChaosDiskWorkload) CloudInitUserdata() (string, error) { }, }, RunCmd: [][]string{ - {"mkdir", "-p", "/mnt/data"}, + {"/usr/local/bin/virtwork-disk-setup.sh"}, {"systemctl", "daemon-reload"}, {"systemctl", "enable", "--now", "virtwork-chaos-disk.service"}, }, @@ -116,7 +121,8 @@ func (w *ChaosDiskWorkload) DataVolumeTemplates() []kubevirtv1.DataVolumeTemplat func (w *ChaosDiskWorkload) ExtraDisks() []kubevirtv1.Disk { return []kubevirtv1.Disk{ { - Name: "datadisk", + Name: "datadisk", + Serial: "virtwork-chdisk", DiskDevice: kubevirtv1.DiskDevice{ Disk: &kubevirtv1.DiskTarget{ Bus: "virtio", diff --git a/internal/workloads/chaos_disk_test.go b/internal/workloads/chaos_disk_test.go index 170cfe2..95a4540 100644 --- a/internal/workloads/chaos_disk_test.go +++ b/internal/workloads/chaos_disk_test.go @@ -115,10 +115,11 @@ var _ = Describe("ChaosDiskWorkload", func() { Expect(dvts[0].Name).To(Equal("virtwork-chaos-disk-data")) }) - It("should have extra disk for data volume", func() { + It("should have extra disk for data volume with serial", func() { disks := w.ExtraDisks() Expect(disks).To(HaveLen(1)) Expect(disks[0].Name).To(Equal("datadisk")) + Expect(disks[0].Serial).To(Equal("virtwork-chdisk")) volumes := w.ExtraVolumes() Expect(volumes).To(HaveLen(1)) @@ -145,7 +146,28 @@ var _ = Describe("ChaosDiskWorkload", func() { Expect(hasPackages).To(BeFalse()) }) - It("should mount data disk and enable service via runcmd", func() { + It("should include disk-setup script in write_files", func() { + result, err := w.CloudInitUserdata() + Expect(err).NotTo(HaveOccurred()) + + parsed := parseYAML(result) + files := parsed["write_files"].([]interface{}) + + var setupContent string + for _, f := range files { + fm := f.(map[string]interface{}) + if fm["path"] == "/usr/local/bin/virtwork-disk-setup.sh" { + setupContent = fm["content"].(string) + break + } + } + Expect(setupContent).NotTo(BeEmpty()) + Expect(setupContent).To(ContainSubstring("virtio-virtwork-chdisk")) + Expect(setupContent).To(ContainSubstring("/mnt/data")) + Expect(setupContent).To(ContainSubstring("mkfs.xfs")) + }) + + It("should run disk-setup script and enable service via runcmd", func() { result, err := w.CloudInitUserdata() Expect(err).NotTo(HaveOccurred()) @@ -166,6 +188,7 @@ var _ = Describe("ChaosDiskWorkload", func() { cmdStrings = append(cmdStrings, s) } + Expect(cmdStrings).To(ContainElement(ContainSubstring("virtwork-disk-setup.sh"))) Expect(cmdStrings).To(ContainElement(ContainSubstring("daemon-reload"))) Expect(cmdStrings).To(ContainElement(ContainSubstring("virtwork-chaos-disk.service"))) }) diff --git a/internal/workloads/database.go b/internal/workloads/database.go index 3b916e7..a7df51d 100644 --- a/internal/workloads/database.go +++ b/internal/workloads/database.go @@ -22,11 +22,27 @@ if [ -f "${MARKER}" ]; then exit 0 fi +# Discover the data disk by virtio serial +DISK="/dev/disk/by-id/virtio-virtwork-dbdisk" +for i in $(seq 1 30); do + [ -e "${DISK}" ] && break + sleep 1 +done +if [ ! -e "${DISK}" ]; then + echo "ERROR: disk ${DISK} not found after 30s" >&2 + exit 1 +fi +REAL_DEV=$(readlink -f "${DISK}") + # Format and mount the data disk if ! mountpoint -q "${DATA_DIR}"; then - mkfs.xfs /dev/vdc - mount /dev/vdc "${DATA_DIR}" - echo '/dev/vdc /var/lib/pgsql/data xfs defaults 0 0' >> /etc/fstab + if ! blkid -o value -s TYPE "${REAL_DEV}" > /dev/null 2>&1; then + mkfs.xfs "${REAL_DEV}" + fi + mount "${REAL_DEV}" "${DATA_DIR}" + if ! grep -q "${DATA_DIR}" /etc/fstab; then + echo "${REAL_DEV} ${DATA_DIR} xfs defaults,nofail 0 0" >> /etc/fstab + fi fi # Set ownership for postgres user @@ -136,7 +152,8 @@ func (w *DatabaseWorkload) DataVolumeTemplates() []kubevirtv1.DataVolumeTemplate func (w *DatabaseWorkload) ExtraDisks() []kubevirtv1.Disk { return []kubevirtv1.Disk{ { - Name: "datadisk", + Name: "datadisk", + Serial: "virtwork-dbdisk", DiskDevice: kubevirtv1.DiskDevice{ Disk: &kubevirtv1.DiskTarget{ Bus: "virtio", diff --git a/internal/workloads/database_test.go b/internal/workloads/database_test.go index e66462c..ef08826 100644 --- a/internal/workloads/database_test.go +++ b/internal/workloads/database_test.go @@ -51,7 +51,7 @@ var _ = Describe("DatabaseWorkload", func() { Expect(paths).To(ContainElement("/usr/local/bin/virtwork-db-setup.sh")) }) - It("should include setup script with initdb and pgbench init", func() { + It("should include setup script with serial discovery and initdb", func() { result, err := w.CloudInitUserdata() Expect(err).NotTo(HaveOccurred()) @@ -67,6 +67,7 @@ var _ = Describe("DatabaseWorkload", func() { } } Expect(setupContent).NotTo(BeEmpty()) + Expect(setupContent).To(ContainSubstring("virtio-virtwork-dbdisk")) Expect(setupContent).To(ContainSubstring("postgresql-setup --initdb")) Expect(setupContent).To(ContainSubstring("pgbench")) Expect(setupContent).To(ContainSubstring("scale")) @@ -142,10 +143,11 @@ var _ = Describe("DatabaseWorkload", func() { Expect(dvts[0].Name).To(Equal("virtwork-database-data")) }) - It("should have extra disk for data volume", func() { + It("should have extra disk for data volume with serial", func() { disks := w.ExtraDisks() Expect(disks).To(HaveLen(1)) Expect(disks[0].Name).To(Equal("datadisk")) + Expect(disks[0].Serial).To(Equal("virtwork-dbdisk")) volumes := w.ExtraVolumes() Expect(volumes).To(HaveLen(1)) diff --git a/internal/workloads/disk.go b/internal/workloads/disk.go index eecc366..c5eabf3 100644 --- a/internal/workloads/disk.go +++ b/internal/workloads/disk.go @@ -91,6 +91,11 @@ func (w *DiskWorkload) CloudInitUserdata() (string, error) { return w.BuildCloudConfig(CloudConfigOpts{ Packages: []string{"fio"}, WriteFiles: []WriteFile{ + { + Path: "/usr/local/bin/virtwork-disk-setup.sh", + Content: diskSetupScript("virtwork-disk", "/mnt/data"), + Permissions: "0755", + }, { Path: "/etc/fio/mixed-rw.fio", Content: fioMixedRWProfile, @@ -108,7 +113,7 @@ func (w *DiskWorkload) CloudInitUserdata() (string, error) { }, }, RunCmd: [][]string{ - {"mkdir", "-p", "/mnt/data"}, + {"/usr/local/bin/virtwork-disk-setup.sh"}, {"systemctl", "daemon-reload"}, {"systemctl", "enable", "--now", "virtwork-disk.service"}, }, @@ -126,7 +131,8 @@ func (w *DiskWorkload) DataVolumeTemplates() []kubevirtv1.DataVolumeTemplateSpec func (w *DiskWorkload) ExtraDisks() []kubevirtv1.Disk { return []kubevirtv1.Disk{ { - Name: "datadisk", + Name: "datadisk", + Serial: "virtwork-disk", DiskDevice: kubevirtv1.DiskDevice{ Disk: &kubevirtv1.DiskTarget{ Bus: "virtio", diff --git a/internal/workloads/disk_test.go b/internal/workloads/disk_test.go index 2fa483d..987f6fc 100644 --- a/internal/workloads/disk_test.go +++ b/internal/workloads/disk_test.go @@ -37,20 +37,21 @@ var _ = Describe("DiskWorkload", func() { Expect(pkgs).To(ContainElement("fio")) }) - It("should include fio profiles in write_files", func() { + It("should include fio profiles and disk-setup script in write_files", func() { result, err := w.CloudInitUserdata() Expect(err).NotTo(HaveOccurred()) parsed := parseYAML(result) files := parsed["write_files"].([]interface{}) - // Should have: mixed-rw.fio, seq-write.fio, systemd unit = 3 files - Expect(files).To(HaveLen(3)) + // Should have: disk-setup.sh, mixed-rw.fio, seq-write.fio, systemd unit = 4 files + Expect(files).To(HaveLen(4)) paths := make([]string, len(files)) for i, f := range files { paths[i] = f.(map[string]interface{})["path"].(string) } + Expect(paths).To(ContainElement("/usr/local/bin/virtwork-disk-setup.sh")) Expect(paths).To(ContainElement("/etc/fio/mixed-rw.fio")) Expect(paths).To(ContainElement("/etc/fio/seq-write.fio")) Expect(paths).To(ContainElement("/etc/systemd/system/virtwork-disk.service")) @@ -62,16 +63,38 @@ var _ = Describe("DiskWorkload", func() { Expect(dvts[0].Name).To(Equal("virtwork-disk-data")) }) - It("should have extra disk for data volume", func() { + It("should have extra disk for data volume with serial", func() { disks := w.ExtraDisks() Expect(disks).To(HaveLen(1)) Expect(disks[0].Name).To(Equal("datadisk")) + Expect(disks[0].Serial).To(Equal("virtwork-disk")) volumes := w.ExtraVolumes() Expect(volumes).To(HaveLen(1)) Expect(volumes[0].Name).To(Equal("datadisk")) }) + It("should include disk-setup script with serial discovery", func() { + result, err := w.CloudInitUserdata() + Expect(err).NotTo(HaveOccurred()) + + parsed := parseYAML(result) + files := parsed["write_files"].([]interface{}) + + var setupContent string + for _, f := range files { + fm := f.(map[string]interface{}) + if fm["path"] == "/usr/local/bin/virtwork-disk-setup.sh" { + setupContent = fm["content"].(string) + break + } + } + Expect(setupContent).NotTo(BeEmpty()) + Expect(setupContent).To(ContainSubstring("virtio-virtwork-disk")) + Expect(setupContent).To(ContainSubstring("/mnt/data")) + Expect(setupContent).To(ContainSubstring("mkfs.xfs")) + }) + It("should not require service", func() { Expect(w.RequiresService()).To(BeFalse()) Expect(w.ServiceSpec()).To(BeNil()) diff --git a/internal/workloads/workload.go b/internal/workloads/workload.go index e9b1764..fa36117 100644 --- a/internal/workloads/workload.go +++ b/internal/workloads/workload.go @@ -4,6 +4,8 @@ package workloads import ( + "fmt" + corev1 "k8s.io/api/core/v1" kubevirtv1 "kubevirt.io/api/core/v1" @@ -116,6 +118,46 @@ func (b *BaseWorkload) VMCount() int { return b.Config.VMCount } +func diskSetupScript(serial, mountPoint string) string { + return fmt.Sprintf(`#!/bin/bash +set -euo pipefail + +DISK="/dev/disk/by-id/virtio-%s" +MOUNT_POINT="%s" + +# Wait for the disk symlink to appear (udev may be slow) +for i in $(seq 1 30); do + [ -e "${DISK}" ] && break + sleep 1 +done + +if [ ! -e "${DISK}" ]; then + echo "ERROR: disk ${DISK} not found after 30s" >&2 + exit 1 +fi + +# Resolve the symlink to the real block device +REAL_DEV=$(readlink -f "${DISK}") + +mkdir -p "${MOUNT_POINT}" + +# Format only if no filesystem exists +if ! blkid -o value -s TYPE "${REAL_DEV}" > /dev/null 2>&1; then + mkfs.xfs "${REAL_DEV}" +fi + +# Mount if not already mounted +if ! mountpoint -q "${MOUNT_POINT}"; then + mount "${REAL_DEV}" "${MOUNT_POINT}" +fi + +# Ensure fstab entry for persistence across reboots +if ! grep -q "${MOUNT_POINT}" /etc/fstab; then + echo "${REAL_DEV} ${MOUNT_POINT} xfs defaults,nofail 0 0" >> /etc/fstab +fi +`, serial, mountPoint) +} + // BuildCloudConfig injects SSH credentials into the given options and delegates // to cloudinit.BuildCloudConfig. Workloads should call this instead of the // package-level function to ensure consistent SSH credential handling.