hypervisor: add hot plugging support for Qemu Q35

Qemu Q35 machine type does not support to hot plug devices
directly on pcie.0, hence we have to find a way to allow users
hot plug N devices.
The only way to hot plug devices in Qemu Q35 is through PCI brdiges.
Each PCI bridge is able to support until 32 devices, therefore we have a
limitation in the amount of devices we can hot plug.
This patch adds support for hot plugging devices on PCI bridges in
pc and q35 machine types.

Signed-off-by: Julio Montes <julio.montes@intel.com>

Author: Julio Montes

api_test.go

@@ -595,6 +595,9 @@ func TestStatusPodSuccessfulStateReady(t *testing.T) {
 		KernelPath:     filepath.Join(testDir, testKernel),
 		ImagePath:      filepath.Join(testDir, testImage),
 		HypervisorPath: filepath.Join(testDir, testHypervisor),
+		DefaultVCPUs:   defaultVCPUs,
+		DefaultMemSz:   defaultMemSzMiB,
+		DefaultBridges: defaultBridges,
 	}
 
 	expectedStatus := PodStatus{
@@ -648,6 +651,9 @@ func TestStatusPodSuccessfulStateRunning(t *testing.T) {
 		KernelPath:     filepath.Join(testDir, testKernel),
 		ImagePath:      filepath.Join(testDir, testImage),
 		HypervisorPath: filepath.Join(testDir, testHypervisor),
+		DefaultVCPUs:   defaultVCPUs,
+		DefaultMemSz:   defaultMemSzMiB,
+		DefaultBridges: defaultBridges,
 	}
 
 	expectedStatus := PodStatus{

bridge.go

@@ -0,0 +1,104 @@
+//
+// Copyright (c) 2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package virtcontainers
+
+import "fmt"
+
+type bridgeType string
+
+const (
+	pciBridge  bridgeType = "pci"
+	pcieBridge            = "pcie"
+)
+
+const pciBridgeMaxCapacity = 30
+
+// Bridge is a bridge where devices can be hot plugged
+type Bridge struct {
+	// Address contains information about devices plugged and its address in the bridge
+	Address map[uint32]string
+
+	// Type is the type of the bridge (pci, pcie, etc)
+	Type bridgeType
+
+	//ID is used to identify the bridge in the hypervisor
+	ID string
+}
+
+// NewBridges creates n new pci(e) bridges depending of the machine type
+func NewBridges(n uint32, machine string) []Bridge {
+	var bridges []Bridge
+	var bt bridgeType
+
+	switch machine {
+	case QemuQ35:
+		// currently only pci bridges are supported
+		// qemu-2.10 will introduce pcie bridges
+		fallthrough
+	case QemuPC:
+		bt = pciBridge
+	default:
+		return nil
+	}
+
+	for i := uint32(0); i < n; i++ {
+		bridges = append(bridges, Bridge{
+			Type:    bt,
+			ID:      fmt.Sprintf("%s-bridge-%d", bt, i),
+			Address: make(map[uint32]string),
+		})
+	}
+
+	return bridges
+}
+
+// addDevice on success adds the device ID to the bridge and return the address
+// where the device was added, otherwise an error is returned
+func (b *Bridge) addDevice(ID string) (uint32, error) {
+	var addr uint32
+
+	// looking for the first available address
+	for i := uint32(1); i <= pciBridgeMaxCapacity; i++ {
+		if _, ok := b.Address[i]; !ok {
+			addr = i
+			break
+		}
+	}
+
+	if addr == 0 {
+		return 0, fmt.Errorf("Unable to hot plug device on bridge: there are not empty slots")
+	}
+
+	// save address and device
+	b.Address[addr] = ID
+	return addr, nil
+}
+
+// removeDevice on success removes the device ID from the bridge and return nil,
+// otherwise an error is returned
+func (b *Bridge) removeDevice(ID string) error {
+	// check if the device was hot plugged in the bridge
+	for addr, devID := range b.Address {
+		if devID == ID {
+			// free address to re-use the same slot with other devices
+			delete(b.Address, addr)
+			return nil
+		}
+	}
+
+	return fmt.Errorf("Unable to hot unplug device %s: not present on bridge", ID)
+}

bridge_test.go

@@ -0,0 +1,65 @@
+//
+// Copyright (c) 2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package virtcontainers
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewBridges(t *testing.T) {
+	assert := assert.New(t)
+	var countBridges uint32 = 1
+
+	bridges := NewBridges(countBridges, "")
+	assert.Nil(bridges)
+
+	bridges = NewBridges(countBridges, QemuQ35)
+	assert.Len(bridges, int(countBridges))
+
+	b := bridges[0]
+	assert.NotEmpty(b.ID)
+	assert.NotNil(b.Address)
+}
+
+func TestAddRemoveDevice(t *testing.T) {
+	assert := assert.New(t)
+	var countBridges uint32 = 1
+
+	// create a bridge
+	bridges := NewBridges(countBridges, "")
+	assert.Nil(bridges)
+	bridges = NewBridges(countBridges, QemuQ35)
+	assert.Len(bridges, int(countBridges))
+
+	// add device
+	devID := "abc123"
+	b := bridges[0]
+	addr, err := b.addDevice(devID)
+	assert.NoError(err)
+	if addr < 1 {
+		assert.Fail("address cannot be less then 1")
+	}
+
+	// remove device
+	err = b.removeDevice("")
+	assert.Error(err)
+
+	err = b.removeDevice(devID)
+	assert.NoError(err)
+}

filesystem.go

@@ -42,6 +42,9 @@ const (
 	// networkFileType represents a network file type (pod only)
 	networkFileType
 
+	// hypervisorFileType represents a hypervisor file type (pod only)
+	hypervisorFileType
+
 	// processFileType represents a process file type
 	processFileType
 
@@ -64,6 +67,9 @@ const stateFile = "state.json"
 // networkFile is the file name storing a pod network.
 const networkFile = "network.json"
 
+// hypervisorFile is the file name storing a hypervisor's state.
+const hypervisorFile = "hypervisor.json"
+
 // processFile is the file name storing a container process.
 const processFile = "process.json"
 
@@ -109,6 +115,10 @@ type resourceStorage interface {
 	fetchPodNetwork(podID string) (NetworkNamespace, error)
 	storePodNetwork(podID string, networkNS NetworkNamespace) error
 
+	// Hypervisor resources
+	fetchHypervisorState(podID string, state interface{}) error
+	storeHypervisorState(podID string, state interface{}) error
+
 	// Container resources
 	storeContainerResource(podID, containerID string, resource podResource, data interface{}) error
 	deleteContainerResources(podID, containerID string, resources []podResource) error
@@ -319,7 +329,7 @@ func (fs *filesystem) fetchDeviceFile(fileData []byte, devices *[]Device) error
 func resourceNeedsContainerID(podSpecific bool, resource podResource) bool {
 
 	switch resource {
-	case lockFileType, networkFileType:
+	case lockFileType, networkFileType, hypervisorFileType:
 		// pod-specific resources
 		return false
 	default:
@@ -342,7 +352,7 @@ func resourceDir(podSpecific bool, podID, containerID string, resource podResour
 	case configFileType:
 		path = configStoragePath
 		break
-	case stateFileType, networkFileType, processFileType, lockFileType, mountsFileType, devicesFileType:
+	case stateFileType, networkFileType, processFileType, lockFileType, mountsFileType, devicesFileType, hypervisorFileType:
 		path = runStoragePath
 		break
 	default:
@@ -378,6 +388,8 @@ func (fs *filesystem) resourceURI(podSpecific bool, podID, containerID string, r
 		filename = stateFile
 	case networkFileType:
 		filename = networkFile
+	case hypervisorFileType:
+		filename = hypervisorFile
 	case processFileType:
 		filename = processFile
 	case lockFileType:
@@ -429,6 +441,7 @@ func (fs *filesystem) commonResourceChecks(podSpecific bool, podID, containerID
 	case configFileType:
 	case stateFileType:
 	case networkFileType:
+	case hypervisorFileType:
 	case processFileType:
 	case mountsFileType:
 	case devicesFileType:
@@ -598,10 +611,23 @@ func (fs *filesystem) fetchPodNetwork(podID string) (NetworkNamespace, error) {
 	return networkNS, nil
 }
 
+func (fs *filesystem) fetchHypervisorState(podID string, state interface{}) error {
+	return fs.fetchResource(true, podID, "", hypervisorFileType, state)
+}
+
 func (fs *filesystem) storePodNetwork(podID string, networkNS NetworkNamespace) error {
 	return fs.storePodResource(podID, networkFileType, networkNS)
 }
 
+func (fs *filesystem) storeHypervisorState(podID string, state interface{}) error {
+	hypervisorFile, _, err := fs.resourceURI(true, podID, "", hypervisorFileType)
+	if err != nil {
+		return err
+	}
+
+	return fs.storeFile(hypervisorFile, state)
+}
+
 func (fs *filesystem) deletePodResources(podID string, resources []podResource) error {
 	if resources == nil {
 		resources = []podResource{configFileType, stateFileType}

hypervisor.go

@@ -44,6 +44,8 @@ const (
 	defaultVCPUs = 1
 	// 2 GiB
 	defaultMemSzMiB = 2048
+
+	defaultBridges = 1
 )
 
 // deviceType describes a virtualized device type.
@@ -161,6 +163,10 @@ type HypervisorConfig struct {
 	// Pod configuration VMConfig.Memory overwrites this.
 	DefaultMemSz uint32
 
+	// DefaultBridges specifies default number of bridges for the VM.
+	// Bridges can be used to hot plug devices
+	DefaultBridges uint32
+
 	// MemPrealloc specifies if the memory should be pre-allocated
 	MemPrealloc bool
 
@@ -203,6 +209,10 @@ func (conf *HypervisorConfig) valid() (bool, error) {
 		conf.DefaultMemSz = defaultMemSzMiB
 	}
 
+	if conf.DefaultBridges == 0 {
+		conf.DefaultBridges = defaultBridges
+	}
+
 	return true, nil
 }
 
@@ -434,7 +444,7 @@ func RunningOnVMM(cpuInfoPath string) (bool, error) {
 // hypervisor is the virtcontainers hypervisor interface.
 // The default hypervisor implementation is Qemu.
 type hypervisor interface {
-	init(config HypervisorConfig) error
+	init(pod *Pod) error
 	createPod(podConfig PodConfig) error
 	startPod(startCh, stopCh chan struct{}) error
 	stopPod() error
@@ -445,4 +455,5 @@ type hypervisor interface {
 	hotplugRemoveDevice(devInfo interface{}, devType deviceType) error
 	getPodConsole(podID string) string
 	capabilities() capabilities
+	getState() interface{}
 }

hypervisor_test.go

@@ -179,6 +179,7 @@ func TestHypervisorConfigDefaults(t *testing.T) {
 		HypervisorPath: "",
 		DefaultVCPUs:   defaultVCPUs,
 		DefaultMemSz:   defaultMemSzMiB,
+		DefaultBridges: defaultBridges,
 	}
 	if reflect.DeepEqual(hypervisorConfig, hypervisorConfigDefaultsExpected) == false {
 		t.Fatal()

mock_hypervisor.go

@@ -19,8 +19,8 @@ package virtcontainers
 type mockHypervisor struct {
 }
 
-func (m *mockHypervisor) init(config HypervisorConfig) error {
-	valid, err := config.valid()
+func (m *mockHypervisor) init(pod *Pod) error {
+	valid, err := pod.config.HypervisorConfig.valid()
 	if valid == false || err != nil {
 		return err
 	}
@@ -69,3 +69,7 @@ func (m *mockHypervisor) hotplugRemoveDevice(devInfo interface{}, devType device
 func (m *mockHypervisor) getPodConsole(podID string) string {
 	return ""
 }
+
+func (m *mockHypervisor) getState() interface{} {
+	return nil
+}

mock_hypervisor_test.go

@@ -25,24 +25,30 @@ import (
 func TestMockHypervisorInit(t *testing.T) {
 	var m *mockHypervisor
 
-	wrongConfig := HypervisorConfig{
-		KernelPath:     "",
-		ImagePath:      "",
-		HypervisorPath: "",
+	pod := &Pod{
+		config: &PodConfig{
+			HypervisorConfig: HypervisorConfig{
+				KernelPath:     "",
+				ImagePath:      "",
+				HypervisorPath: "",
+			},
+		},
 	}
 
-	err := m.init(wrongConfig)
+	// wrong config
+	err := m.init(pod)
 	if err == nil {
 		t.Fatal()
 	}
 
-	rightConfig := HypervisorConfig{
+	pod.config.HypervisorConfig = HypervisorConfig{
 		KernelPath:     fmt.Sprintf("%s/%s", testDir, testKernel),
 		ImagePath:      fmt.Sprintf("%s/%s", testDir, testImage),
 		HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor),
 	}
 
-	err = m.init(rightConfig)
+	// right config
+	err = m.init(pod)
 	if err != nil {
 		t.Fatal(err)
 	}