diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index d6fd7a1..e83dc1b 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -467,6 +467,10 @@ "ImportPath": "github.com/ugorji/go/codec", "Rev": "b94837a2404ab90efe9289e77a70694c355739cb" }, + { + "ImportPath": "github.com/vbatts/qcow2", + "Rev": "cbec6a0f73bd869d047ef2fdf6afa464df574d23" + }, { "ImportPath": "github.com/vincent-petithory/dataurl", "Rev": "9a301d65acbb728fcc3ace14f45f511a4cfeea9c" diff --git a/README.md b/README.md index e016235..3d45e0d 100644 --- a/README.md +++ b/README.md @@ -138,28 +138,33 @@ Accessing the newly created CoreOS instance is just a few more clicks away... ## simple usage recipe: a **docker** and **rkt** playground ### create a volume to store your persistent data - + > the step bellow requires `qemu-img`to be present in you macOS host + > one way to achieve that is having it installed through + > [homebrew's](http://brew.sh) by issuing `❯❯❯ brew install qemu` ``` - ❯❯❯ dd if=/dev/zero of=var_lib_docker.img bs=1G count=16 + ❯❯❯ qemu-img create -f qcow2 var_lib_docker.img.qcow2 16G ``` -> will become `/var/lib/{docker|rkt}`. in this example case we created a volume -> with 16GB. + > will become `/var/lib/{docker|rkt}`. in this example case we created a + > **QCow2** volume with 16GB. -### *format* it + **Raw** volumes were the default until version + **[0.7.12](https://github.com/TheNewNormal/corectl/releases/tag/v0.7.12)**. + They are still supported but become a deprecated feature that may disappear + some point in the future. - ``` - ❯❯❯ /usr/local/Cellar/e2fsprogs/1.42.12/sbin/mke2fs -b 1024 -i 1024 -t ext4 -m0 -F var_lib_docker.img - ``` - > requires [homebrew's](http://brew.sh) e2fsprogs package installed. - > - > `❯❯❯ brew install e2fsprogs` - -### *label* it +### *format* and label it + > we'll format and label the newly create volume from within a transient VM + > as it's the simplest way. We're formatting it with `ext4` but you can choose + > any filesystem you like assuming it is a CoreOS supported one. ``` - ❯❯❯ /usr/local/Cellar/e2fsprogs/1.42.12/sbin/e2label var_lib_docker.img rkthdd + ❯❯❯ corectl run --name foo --volume=var_lib_docker.img.qcow2 + ❯❯❯ corectl ssh foo "sudo mke2fs -b 1024 -i 1024 -t ext4 -m0 /dev/vda && \ + sudo e2label /dev/vda rkthdd " + ❯❯❯ corectl halt foo ``` - here, we labeled our volume `rkthdd` which is the *signature* that our + + above, we labeled our volume `rkthdd` which is the *signature* that our [*recipe*](cloud-init/docker-only-with-persistent-storage.txt) expects. >by relying in *labels* for volume identification we get around the issues we'd diff --git a/components/server/qcow2.go b/components/server/qcow2.go new file mode 100644 index 0000000..697d3b0 --- /dev/null +++ b/components/server/qcow2.go @@ -0,0 +1,100 @@ +// Copyright (c) 2016 by António Meireles . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package server + +import ( + "bytes" + "encoding/binary" + "fmt" + "os" + + "github.com/helm/helm/log" + "github.com/vbatts/qcow2" +) + +var ErrFileIsNotQCOW2 = fmt.Errorf("File doesn't appear to be a qcow one") + +// adapted from github.com/vbatts/qcow2/cmd/qcow2-info/main.go +func ValidateQcow2(fh *os.File) (err error) { + var size int + + buf := make([]byte, qcow2.V2HeaderSize) + + if size, err = fh.Read(buf); err != nil { + return + } + + if size >= qcow2.V2HeaderSize && bytes.Compare(buf[:4], qcow2.Magic) != 0 { + log.Debug("%q: Does not appear to be qcow file %#v %#v", + fh.Name(), buf[:4], qcow2.Magic) + return ErrFileIsNotQCOW2 + } + + q := qcow2.Header{ + Version: qcow2.Version(be32(buf[4:8])), + BackingFileOffset: be64(buf[8:16]), + BackingFileSize: be32(buf[16:20]), + ClusterBits: be32(buf[20:24]), + Size: be64(buf[24:32]), + CryptMethod: qcow2.CryptMethod(be32(buf[32:36])), + L1Size: be32(buf[36:40]), + L1TableOffset: be64(buf[40:48]), + RefcountTableOffset: be64(buf[48:56]), + RefcountTableClusters: be32(buf[56:60]), + NbSnapshots: be32(buf[60:64]), + SnapshotsOffset: be64(buf[64:72]), + HeaderLength: 72, // v2 this is a standard length + } + + if q.Version == 3 { + if size, err = fh.Read(buf[:qcow2.V3HeaderSize]); err != nil { + return fmt.Errorf("(qcow2) error validating %q: %s", + fh.Name(), err) + } + if size < qcow2.V3HeaderSize { + return fmt.Errorf("(qcow2) error validating %q: short read", + fh.Name()) + } + + q.IncompatibleFeatures = be32(buf[0:8]) + q.CompatibleFeatures = be32(buf[8:16]) + q.AutoclearFeatures = be32(buf[16:24]) + q.RefcountOrder = be32(buf[24:28]) + q.HeaderLength = be32(buf[28:32]) + } + if log.IsDebugging { + log.Info("%#v\n", q) + log.Info("IncompatibleFeatures: %b\n", q.IncompatibleFeatures) + log.Info("CompatibleFeatures: %b\n", q.CompatibleFeatures) + } + // Process the extension header data + buf = make([]byte, q.HeaderLength) + if size, err = fh.Read(buf); err != nil { + return fmt.Errorf("(qcow2) error validating %q: %s", fh.Name(), err) + } + if size < q.HeaderLength { + return fmt.Errorf("(qcow2) error validating %q: short read", fh.Name()) + } + return +} + +func be32(b []byte) int { + return int(binary.BigEndian.Uint32(b)) +} + +func be64(b []byte) int64 { + return int64(binary.BigEndian.Uint64(b)) +} diff --git a/components/server/run.go b/components/server/run.go index 113c12a..cfe7bec 100644 --- a/components/server/run.go +++ b/components/server/run.go @@ -66,8 +66,8 @@ type ( } // StorageDevice ... StorageDevice struct { - Slot int - Type, Path string + Slot, Format int + Type, Path string } // StorageAssets ... StorageAssets struct { @@ -78,13 +78,12 @@ type ( const ( _ = iota Raw + Qcow2 Tap - HDD = "HDD" - CDROM = "CDROM" - Local = "localfs" - Remote = "URL" - Attached = true - Detached = false + HDD = "HDD" + CDROM = "CDROM" + Local = "localfs" + Remote = "URL" ) var ServerTimeout = 25 * time.Second @@ -114,7 +113,11 @@ func (vm *VMInfo) ValidateCDROM(path string) (err error) { // ValidateVolumes ... func (vm *VMInfo) ValidateVolumes(volumes []string, root bool) (err error) { - var abs string + var ( + abs string + fh *os.File + format = Qcow2 + ) for _, j := range volumes { if j != "" { @@ -124,9 +127,26 @@ func (vm *VMInfo) ValidateVolumes(volumes []string, root bool) (err error) { if abs, err = filepath.Abs(j); err != nil { return } - if !strings.HasSuffix(j, ".img") { - return fmt.Errorf("Aborting: --volume payload MUST end"+ - " in '.img' ('%s' doesn't)", j) + if fh, err = os.Open(j); err != nil { + return + } + defer fh.Close() + if err = ValidateQcow2(fh); err != nil { + if err != ErrFileIsNotQCOW2 { + return + } + log.Warn("using Raw formated volumes is a deprecated feature " + + "that may become unsupported in the future. Please " + + "consider moving to QCOW2 ones") + format = Raw + err = nil + } + if format == Raw { + // to be consistent with previous behaviour + if !strings.HasSuffix(j, ".img") { + return fmt.Errorf("Aborting: --volume payload MUST end"+ + " in '.img' ('%s' doesn't)", j) + } } // check atomicity reply := &RPCreply{} @@ -157,7 +177,7 @@ func (vm *VMInfo) ValidateVolumes(volumes []string, root bool) (err error) { } } vm.Storage.HardDrives[strconv.Itoa(slot)] = - StorageDevice{Type: HDD, Slot: slot, Path: abs} + StorageDevice{Type: HDD, Format: format, Slot: slot, Path: abs} if root { vm.Root = slot } @@ -295,8 +315,15 @@ func (vm *VMInfo) assembleBootPayload() (xArgs []string, err error) { } for _, v := range vm.Storage.HardDrives { - instr = append(instr, "-s", fmt.Sprintf("4:%d,virtio-blk,%s", - v.Slot, v.Path)) + switch v.Format { + case Raw: + instr = append(instr, "-s", fmt.Sprintf("4:%d,virtio-blk,%s", + v.Slot, v.Path)) + case Qcow2: + instr = append(instr, "-s", + fmt.Sprintf("4:%d,virtio-blk,file://%s,format=qcow", + v.Slot, v.Path)) + } } return []string{strings.Join(instr, " "), @@ -412,11 +439,17 @@ func (volumes *StorageAssets) PrettyPrint(root int) { fmt.Printf(" /dev/cdrom%v\t%s\n", a, b.Path) } for a, b := range volumes.HardDrives { + format := "raw" i, _ := strconv.Atoi(a) + if b.Format == Qcow2 { + format = "qcow2" + } if i != root { - fmt.Printf(" /dev/vd%v\t%s\n", string(i+'a'), b.Path) + fmt.Printf(" /dev/vd%v\t%s,format=%s\n", string(i+'a'), + b.Path, format) } else { - fmt.Printf(" /,/dev/vd%v\t%s\n", string(i+'a'), b.Path) + fmt.Printf(" /,/dev/vd%v\t%s,format=%s\n", string(i+'a'), + b.Path, format) } } } diff --git a/vendor/github.com/vbatts/qcow2/LICENSE b/vendor/github.com/vbatts/qcow2/LICENSE new file mode 100644 index 0000000..8ba5491 --- /dev/null +++ b/vendor/github.com/vbatts/qcow2/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2015 Vincent Batts, Raleigh, NC, USA + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/vbatts/qcow2/README.md b/vendor/github.com/vbatts/qcow2/README.md new file mode 100644 index 0000000..2e69515 --- /dev/null +++ b/vendor/github.com/vbatts/qcow2/README.md @@ -0,0 +1,13 @@ +# qcow2 + +WIP bindings for qcow2 file format + +## installing + +```bash +go get github.com/vbatts/qcow2/cmd/qcow2-info +``` + +## License + +See [LICENSE](./LICENSE) diff --git a/vendor/github.com/vbatts/qcow2/cmd/qcow2-info/main.go b/vendor/github.com/vbatts/qcow2/cmd/qcow2-info/main.go new file mode 100644 index 0000000..6a31f62 --- /dev/null +++ b/vendor/github.com/vbatts/qcow2/cmd/qcow2-info/main.go @@ -0,0 +1,114 @@ +package main + +import ( + "bytes" + "encoding/binary" + "flag" + "fmt" + "os" + + "github.com/vbatts/qcow2" +) + +func main() { + flag.Parse() + + for _, arg := range flag.Args() { + fh, err := os.Open(arg) + if err != nil { + fmt.Fprintf(os.Stderr, "[ERR] %q: %s\n", arg, err) + os.Exit(1) + } + defer fh.Close() + + buf := make([]byte, qcow2.V2HeaderSize) + size, err := fh.Read(buf) + if err != nil { + fmt.Fprintf(os.Stderr, "[ERR] %q: %s\n", arg, err) + os.Exit(1) + } + if size < qcow2.V2HeaderSize { + fmt.Fprintf(os.Stderr, "[ERR] %q: short read\n", arg) + os.Exit(1) + } + + if bytes.Compare(buf[:4], qcow2.Magic) != 0 { + fmt.Fprintf(os.Stderr, "[ERR] %q: Does not appear to be qcow file %#v %#v\n", arg, buf[:4], qcow2.Magic) + os.Exit(1) + } + + q := qcow2.Header{ + Version: qcow2.Version(be32(buf[4:8])), + BackingFileOffset: be64(buf[8:16]), + BackingFileSize: be32(buf[16:20]), + ClusterBits: be32(buf[20:24]), + Size: be64(buf[24:32]), + CryptMethod: qcow2.CryptMethod(be32(buf[32:36])), + L1Size: be32(buf[36:40]), + L1TableOffset: be64(buf[40:48]), + RefcountTableOffset: be64(buf[48:56]), + RefcountTableClusters: be32(buf[56:60]), + NbSnapshots: be32(buf[60:64]), + SnapshotsOffset: be64(buf[64:72]), + HeaderLength: 72, // v2 this is a standard length + } + + if q.Version == 3 { + size, err := fh.Read(buf[:qcow2.V3HeaderSize]) + if err != nil { + fmt.Fprintf(os.Stderr, "[ERR] %q: %s\n", arg, err) + os.Exit(1) + } + if size < qcow2.V3HeaderSize { + fmt.Fprintf(os.Stderr, "[ERR] %q: short read\n", arg) + os.Exit(1) + } + + q.IncompatibleFeatures = be32(buf[0:8]) + q.CompatibleFeatures = be32(buf[8:16]) + q.AutoclearFeatures = be32(buf[16:24]) + q.RefcountOrder = be32(buf[24:28]) + q.HeaderLength = be32(buf[28:32]) + } + fmt.Printf("%#v\n", q) + fmt.Printf("IncompatibleFeatures: %b\n", q.IncompatibleFeatures) + fmt.Printf("CompatibleFeatures: %b\n", q.CompatibleFeatures) + + // Process the extension header data + buf = make([]byte, q.HeaderLength) + size, err = fh.Read(buf) + if err != nil { + fmt.Fprintf(os.Stderr, "[ERR] %q: %s\n", arg, err) + os.Exit(1) + } + if size < q.HeaderLength { + fmt.Fprintf(os.Stderr, "[ERR] %q: short read\n", arg) + os.Exit(1) + } + for { + t := qcow2.HeaderExtensionType(be32(buf[:4])) + if t == qcow2.HdrExtEndOfArea { + break + } + exthdr := qcow2.ExtHeader{ + Type: t, + Size: be32(buf[4:8]), + } + // XXX this may need a copy(), so the slice resuse doesn't corrupt + exthdr.Data = buf[8 : 8+exthdr.Size] + q.ExtHeaders = append(q.ExtHeaders, exthdr) + + round := exthdr.Size % 8 + buf = buf[8+exthdr.Size+round:] + } + + } +} + +func be32(b []byte) int { + return int(binary.BigEndian.Uint32(b)) +} + +func be64(b []byte) int64 { + return int64(binary.BigEndian.Uint64(b)) +} diff --git a/vendor/github.com/vbatts/qcow2/qcow2.go b/vendor/github.com/vbatts/qcow2/qcow2.go new file mode 100644 index 0000000..fbe8c04 --- /dev/null +++ b/vendor/github.com/vbatts/qcow2/qcow2.go @@ -0,0 +1,69 @@ +package qcow2 + +var ( + // Magic is the front of the file fingerprint + Magic = []byte{0x51, 0x46, 0x49, 0xFB} + + // V2HeaderSize is the image header at the beginning of the file + V2HeaderSize = 72 + + // V3HeaderSize is directly following the v2 header, up to 104 + V3HeaderSize = 104 - V2HeaderSize +) + +type ( + // Version number of this image. Valid versions are 2 or 3 + Version int + + // CryptMethod is whether no encryption (0), or AES encryption (1) + CryptMethod int + + // HeaderExtensionType indicators the the entries in the optional header area + HeaderExtensionType int +) + +const ( + HdrExtEndOfArea HeaderExtensionType = 0x00000000 + HdrExtBackingFileFormat HeaderExtensionType = 0xE2792ACA + HdrExtFeatureNameTable HeaderExtensionType = 0x6803f857 // TODO needs processing for feature name table + // any thing else is "other" and can be ignored +) + +func (qcm CryptMethod) String() string { + if qcm == 1 { + return "AES" + } + return "none" +} + +type Header struct { + // magic [:4] + Version Version // [4:8] + BackingFileOffset int64 // [8:16] + BackingFileSize int // [16:20] + ClusterBits int // [20:24] + Size int64 // [24:32] + CryptMethod CryptMethod // [32:36] + L1Size int // [36:40] + L1TableOffset int64 // [40:48] + RefcountTableOffset int64 // [48:56] + RefcountTableClusters int // [56:60] + NbSnapshots int // [60:64] + SnapshotsOffset int64 // [64:72] + + // v3 + IncompatibleFeatures int // [72:80] bitmask + CompatibleFeatures int // [80:88] bitmask + AutoclearFeatures int // [88:96] bitmask + RefcountOrder int // [96:100] + HeaderLength int // [100:104] + + // Header extensions + ExtHeaders []ExtHeader +} + +type ExtHeader struct { + Type HeaderExtensionType + Size int + Data []byte +} diff --git a/vendor/github.com/vbatts/qcow2/qcow2_test.go b/vendor/github.com/vbatts/qcow2/qcow2_test.go new file mode 100644 index 0000000..57a37fc --- /dev/null +++ b/vendor/github.com/vbatts/qcow2/qcow2_test.go @@ -0,0 +1,146 @@ +package qcow2 + +import ( + "bufio" + "bytes" + "compress/gzip" + "encoding/binary" + "os" + "testing" +) + +/* +qemu-img create -f qcow2 file.qcow2 100m +sudo modprobe nbd max_part=63 +sudo qemu-nbd -f qcow2 -c /dev/nbd0 file.qcow2 +sudo mkfs.ext2 /dev/nbd0 +mkdir -p file/ +sudo mount /dev/nbd0 file/ +sudo umount file/ +sudo qemu-nbd -d /dev/nbd0 +sudo qemu-img snapshot -c base file.qcow2 +sudo qemu-nbd -f qcow2 -c /dev/nbd0 file.qcow2 +sudo mount /dev/nbd0 file +echo Howdy | sudo dd of=file/hello.txt +sudo umount file/ +sudo qemu-nbd -d /dev/nbd0 +sudo qemu-img snapshot -c hello file.qcow2 +sudo qemu-nbd -f qcow2 -c /dev/nbd0 file.qcow2 +sudo mount /dev/nbd0 file +sudo rm file/hello.txt +sudo umount file/ +sudo qemu-nbd -d /dev/nbd0 +ls -lsh ./file.qcow2 +# 4.9M -rw-r--r--. 1 vbatts vbatts 5.1M Sep 3 13:38 file.qcow2 +qcow2-info ./file.qcow2 +# image: ./file.qcow2 +# file format: qcow2 +# virtual size: 100M (104857600 bytes) +# disk size: 4.8M +# cluster_size: 65536 +# Snapshot list: +# ID TAG VM SIZE DATE VM CLOCK +# 1 base 0 2015-09-03 13:36:55 00:00:00.000 +# 2 hello 0 2015-09-03 13:38:07 00:00:00.000 +# Format specific information: +# compat: 1.1 +# lazy refcounts: false +# refcount bits: 16 +# corrupt: false +gzip -9 file.qcow2 > file.qcow2.gz +*/ +var testQcowFile = "./testdata/file.qcow2.gz" + +func TestHeader(t *testing.T) { + f, err := os.Open(testQcowFile) + if err != nil { + t.Fatal(err) + } + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + rdr := bufio.NewReader(gz) + + buf := make([]byte, V2HeaderSize) + size, err := rdr.Read(buf) + if err != nil { + t.Fatal(err) + } + if size < V2HeaderSize { + t.Fatal(err) + } + + if bytes.Compare(buf[:4], Magic) != 0 { + t.Fatalf("[ERR] Does not appear to be qcow file %#v %#v\n", buf[:4], Magic) + } + + q := Header{ + Version: Version(be32(buf[4:8])), + BackingFileOffset: be64(buf[8:16]), + BackingFileSize: be32(buf[16:20]), + ClusterBits: be32(buf[20:24]), + Size: be64(buf[24:32]), + CryptMethod: CryptMethod(be32(buf[32:36])), + L1Size: be32(buf[36:40]), + L1TableOffset: be64(buf[40:48]), + RefcountTableOffset: be64(buf[48:56]), + RefcountTableClusters: be32(buf[56:60]), + NbSnapshots: be32(buf[60:64]), + SnapshotsOffset: be64(buf[64:72]), + HeaderLength: 72, // v2 this is a standard length + } + + if q.Version == 3 { + size, err := rdr.Read(buf[:V3HeaderSize]) + if err != nil { + t.Fatal(err) + } + if size < V3HeaderSize { + t.Fatalf("short read") + } + + q.IncompatibleFeatures = be32(buf[0:8]) + q.CompatibleFeatures = be32(buf[8:16]) + q.AutoclearFeatures = be32(buf[16:24]) + q.RefcountOrder = be32(buf[24:28]) + q.HeaderLength = be32(buf[28:32]) + } + t.Logf("%#v", q) + + // Process the extension header data + buf = make([]byte, q.HeaderLength) + size, err = rdr.Read(buf) + if err != nil { + t.Fatal(err) + } + if size < q.HeaderLength { + t.Fatalf("short read") + } + for { + t := HeaderExtensionType(be32(buf[:4])) + if t == HdrExtEndOfArea { + break + } + exthdr := ExtHeader{ + Type: t, + Size: be32(buf[4:8]), + } + // XXX this may need a copy(), so the slice resuse doesn't corrupt + exthdr.Data = buf[8 : 8+exthdr.Size] + q.ExtHeaders = append(q.ExtHeaders, exthdr) + + round := exthdr.Size % 8 + buf = buf[8+exthdr.Size+round:] + } + + // TODO at this point we can do some assertions on the `q` values +} + +func be32(b []byte) int { + return int(binary.BigEndian.Uint32(b)) +} + +func be64(b []byte) int64 { + return int64(binary.BigEndian.Uint64(b)) +} diff --git a/vendor/github.com/vbatts/qcow2/testdata/file.qcow2.gz b/vendor/github.com/vbatts/qcow2/testdata/file.qcow2.gz new file mode 100644 index 0000000..1ad0b02 Binary files /dev/null and b/vendor/github.com/vbatts/qcow2/testdata/file.qcow2.gz differ