add support for QCow2 volumes

README.md

@@ -138,28 +138,33 @@ Accessing the newly created CoreOS instance is just a few more clicks away...
 ## simple usage recipe: a **docker** and **rkt** playground
 
 ### create a volume to store your persistent data
-
+  > the step bellow requires `qemu-img`to be present in you macOS host
+  > one way to achieve that is having it installed through
+  > [homebrew's](http://brew.sh) by issuing `❯❯❯ brew install qemu`
   ```
-  ❯❯❯ dd if=/dev/zero of=var_lib_docker.img  bs=1G count=16
+  ❯❯❯ qemu-img create -f qcow2 var_lib_docker.img.qcow2 16G
   ```
-> will become  `/var/lib/{docker|rkt}`. in this example case we created a volume
-> with 16GB.
+  > will become `/var/lib/{docker|rkt}`. in this example case we created a
+  > **QCow2** volume with 16GB.
 
-### *format* it
+  **Raw** volumes were the default until version
+  **[0.7.12](https://github.com/TheNewNormal/corectl/releases/tag/v0.7.12)**.
+  They are still supported but become a deprecated feature that may disappear
+  some point in the future.
 
-  ```
-  ❯❯❯ /usr/local/Cellar/e2fsprogs/1.42.12/sbin/mke2fs -b 1024 -i 1024 -t ext4 -m0 -F var_lib_docker.img
-  ```
-  > requires [homebrew's](http://brew.sh) e2fsprogs package installed.
-  >
-  > `❯❯❯ brew install e2fsprogs`
-
-### *label* it
+### *format* and label it
+  > we'll format and label the newly create volume from within a transient VM
+  > as it's the simplest way. We're formatting it with `ext4` but you can choose
+  > any filesystem you like assuming it is a CoreOS supported one.
 
   ```
-  ❯❯❯ /usr/local/Cellar/e2fsprogs/1.42.12/sbin/e2label var_lib_docker.img rkthdd
+  ❯❯❯ corectl run  --name foo --volume=var_lib_docker.img.qcow2
+  ❯❯❯ corectl ssh foo "sudo mke2fs -b 1024 -i 1024 -t ext4 -m0 /dev/vda && \
+        sudo e2label /dev/vda rkthdd "
+  ❯❯❯ corectl halt foo
   ```
-  here, we labeled our volume `rkthdd` which is the *signature* that our
+
+  above, we labeled our volume `rkthdd` which is the *signature* that our
   [*recipe*](cloud-init/docker-only-with-persistent-storage.txt) expects.
 
   >by relying in *labels* for volume identification we get around the issues we'd

components/server/qcow2.go

@@ -0,0 +1,100 @@
+// Copyright (c) 2016 by António Meireles  <antonio.meireles@reformi.st>.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package server
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"os"
+
+	"github.com/helm/helm/log"
+	"github.com/vbatts/qcow2"
+)
+
+var ErrFileIsNotQCOW2 = fmt.Errorf("File doesn't appear to be a qcow one")
+
+// adapted from github.com/vbatts/qcow2/cmd/qcow2-info/main.go
+func ValidateQcow2(fh *os.File) (err error) {
+	var size int
+
+	buf := make([]byte, qcow2.V2HeaderSize)
+
+	if size, err = fh.Read(buf); err != nil {
+		return
+	}
+
+	if size >= qcow2.V2HeaderSize && bytes.Compare(buf[:4], qcow2.Magic) != 0 {
+		log.Debug("%q: Does not appear to be qcow file %#v %#v",
+			fh.Name(), buf[:4], qcow2.Magic)
+		return ErrFileIsNotQCOW2
+	}
+
+	q := qcow2.Header{
+		Version:               qcow2.Version(be32(buf[4:8])),
+		BackingFileOffset:     be64(buf[8:16]),
+		BackingFileSize:       be32(buf[16:20]),
+		ClusterBits:           be32(buf[20:24]),
+		Size:                  be64(buf[24:32]),
+		CryptMethod:           qcow2.CryptMethod(be32(buf[32:36])),
+		L1Size:                be32(buf[36:40]),
+		L1TableOffset:         be64(buf[40:48]),
+		RefcountTableOffset:   be64(buf[48:56]),
+		RefcountTableClusters: be32(buf[56:60]),
+		NbSnapshots:           be32(buf[60:64]),
+		SnapshotsOffset:       be64(buf[64:72]),
+		HeaderLength:          72, // v2 this is a standard length
+	}
+
+	if q.Version == 3 {
+		if size, err = fh.Read(buf[:qcow2.V3HeaderSize]); err != nil {
+			return fmt.Errorf("(qcow2) error validating %q: %s",
+				fh.Name(), err)
+		}
+		if size < qcow2.V3HeaderSize {
+			return fmt.Errorf("(qcow2) error validating %q: short read",
+				fh.Name())
+		}
+
+		q.IncompatibleFeatures = be32(buf[0:8])
+		q.CompatibleFeatures = be32(buf[8:16])
+		q.AutoclearFeatures = be32(buf[16:24])
+		q.RefcountOrder = be32(buf[24:28])
+		q.HeaderLength = be32(buf[28:32])
+	}
+	if log.IsDebugging {
+		log.Info("%#v\n", q)
+		log.Info("IncompatibleFeatures: %b\n", q.IncompatibleFeatures)
+		log.Info("CompatibleFeatures: %b\n", q.CompatibleFeatures)
+	}
+	// Process the extension header data
+	buf = make([]byte, q.HeaderLength)
+	if size, err = fh.Read(buf); err != nil {
+		return fmt.Errorf("(qcow2) error validating %q: %s", fh.Name(), err)
+	}
+	if size < q.HeaderLength {
+		return fmt.Errorf("(qcow2) error validating %q: short read", fh.Name())
+	}
+	return
+}
+
+func be32(b []byte) int {
+	return int(binary.BigEndian.Uint32(b))
+}
+
+func be64(b []byte) int64 {
+	return int64(binary.BigEndian.Uint64(b))
+}

components/server/run.go

@@ -66,8 +66,8 @@ type (
 	}
 	// StorageDevice ...
 	StorageDevice struct {
-		Slot       int
-		Type, Path string
+		Slot, Format int
+		Type, Path   string
 	}
 	// StorageAssets ...
 	StorageAssets struct {
@@ -78,13 +78,12 @@ type (
 const (
 	_ = iota
 	Raw
+	Qcow2
 	Tap
-	HDD      = "HDD"
-	CDROM    = "CDROM"
-	Local    = "localfs"
-	Remote   = "URL"
-	Attached = true
-	Detached = false
+	HDD    = "HDD"
+	CDROM  = "CDROM"
+	Local  = "localfs"
+	Remote = "URL"
 )
 
 var ServerTimeout = 25 * time.Second
@@ -114,7 +113,11 @@ func (vm *VMInfo) ValidateCDROM(path string) (err error) {
 
 // ValidateVolumes ...
 func (vm *VMInfo) ValidateVolumes(volumes []string, root bool) (err error) {
-	var abs string
+	var (
+		abs    string
+		fh     *os.File
+		format = Qcow2
+	)
 
 	for _, j := range volumes {
 		if j != "" {
@@ -124,9 +127,26 @@ func (vm *VMInfo) ValidateVolumes(volumes []string, root bool) (err error) {
 			if abs, err = filepath.Abs(j); err != nil {
 				return
 			}
-			if !strings.HasSuffix(j, ".img") {
-				return fmt.Errorf("Aborting: --volume payload MUST end"+
-					" in '.img' ('%s' doesn't)", j)
+			if fh, err = os.Open(j); err != nil {
+				return
+			}
+			defer fh.Close()
+			if err = ValidateQcow2(fh); err != nil {
+				if err != ErrFileIsNotQCOW2 {
+					return
+				}
+				log.Warn("using Raw formated volumes is a deprecated feature " +
+					"that may become unsupported in the future. Please " +
+					"consider moving to QCOW2 ones")
+				format = Raw
+				err = nil
+			}
+			if format == Raw {
+				// to be consistent with previous behaviour
+				if !strings.HasSuffix(j, ".img") {
+					return fmt.Errorf("Aborting: --volume payload MUST end"+
+						" in '.img' ('%s' doesn't)", j)
+				}
 			}
 			// check atomicity
 			reply := &RPCreply{}
@@ -157,7 +177,7 @@ func (vm *VMInfo) ValidateVolumes(volumes []string, root bool) (err error) {
 				}
 			}
 			vm.Storage.HardDrives[strconv.Itoa(slot)] =
-				StorageDevice{Type: HDD, Slot: slot, Path: abs}
+				StorageDevice{Type: HDD, Format: format, Slot: slot, Path: abs}
 			if root {
 				vm.Root = slot
 			}
@@ -295,8 +315,15 @@ func (vm *VMInfo) assembleBootPayload() (xArgs []string, err error) {
 	}
 
 	for _, v := range vm.Storage.HardDrives {
-		instr = append(instr, "-s", fmt.Sprintf("4:%d,virtio-blk,%s",
-			v.Slot, v.Path))
+		switch v.Format {
+		case Raw:
+			instr = append(instr, "-s", fmt.Sprintf("4:%d,virtio-blk,%s",
+				v.Slot, v.Path))
+		case Qcow2:
+			instr = append(instr, "-s",
+				fmt.Sprintf("4:%d,virtio-blk,file://%s,format=qcow",
+					v.Slot, v.Path))
+		}
 	}
 
 	return []string{strings.Join(instr, " "),
@@ -412,11 +439,17 @@ func (volumes *StorageAssets) PrettyPrint(root int) {
 			fmt.Printf("   /dev/cdrom%v\t%s\n", a, b.Path)
 		}
 		for a, b := range volumes.HardDrives {
+			format := "raw"
 			i, _ := strconv.Atoi(a)
+			if b.Format == Qcow2 {
+				format = "qcow2"
+			}
 			if i != root {
-				fmt.Printf("   /dev/vd%v\t%s\n", string(i+'a'), b.Path)
+				fmt.Printf("   /dev/vd%v\t%s,format=%s\n", string(i+'a'),
+					b.Path, format)
 			} else {
-				fmt.Printf("   /,/dev/vd%v\t%s\n", string(i+'a'), b.Path)
+				fmt.Printf("   /,/dev/vd%v\t%s,format=%s\n", string(i+'a'),
+					b.Path, format)
 			}
 		}
 	}