Skip to content
This repository was archived by the owner on Jul 18, 2025. It is now read-only.

Multiple -f #116

Closed
joshwget wants to merge 4 commits into from
Closed

Multiple -f #116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
013e101
Multiple -f
joshwget Dec 3, 2015
5138735
Updated README and example program
joshwget Dec 19, 2015
84faa69
Look for "docker-compose.override.yml" by default
joshwget Dec 23, 2015
0db6452
Added integration test for multiple files with multiple services
joshwget Dec 26, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ import (
func main() {
project, err := docker.NewProject(&docker.Context{
Context: project.Context{
ComposeFile: "docker-compose.yml",
ProjectName: "my-compose",
ComposeFiles: []string{"docker-compose.yml"},
ProjectName: "my-compose",
},
})

Expand Down
19 changes: 15 additions & 4 deletions cli/command/command.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package command

import (
"os"

"github.com/codegangsta/cli"
"github.com/docker/libcompose/cli/app"
"github.com/docker/libcompose/project"
Expand Down Expand Up @@ -213,10 +215,10 @@ func CommonFlags() []cli.Flag {
cli.BoolFlag{
Name: "verbose,debug",
},
cli.StringFlag{
cli.StringSliceFlag{
Name: "file,f",
Usage: "Specify an alternate compose file (default: docker-compose.yml)",
Value: "docker-compose.yml",
Usage: "Specify one or more alternate compose files (default: docker-compose.yml)",
Value: &cli.StringSlice{},
EnvVar: "COMPOSE_FILE",
},
cli.StringFlag{
Expand All @@ -228,7 +230,16 @@ func CommonFlags() []cli.Flag {

// Populate updates the specified project context based on command line arguments and subcommands.
func Populate(context *project.Context, c *cli.Context) {
context.ComposeFile = c.GlobalString("file")
if len(c.GlobalStringSlice("file")) == 0 {
Copy link
Copy Markdown
Contributor

@vdemeester vdemeester Dec 26, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would have written like this (because I don't like else's 😝), but not sure if it's better or not...

    context.ComposeFiles = c.GlobalStringSlice("file")

    if len(context.ComposeFiles) == 0 {
        context.ComposeFiles = []string{"docker-compose.yml"}
        if _, err := os.Stat("docker-compose.override.yml"); err == nil {
            context.ComposeFiles = append(context.ComposeFiles, "docker-compose.override.yml")
        }
    }

if _, err := os.Stat("docker-compose.override.yml"); err == nil {
context.ComposeFiles = []string{"docker-compose.yml", "docker-compose.override.yml"}
} else {
context.ComposeFiles = []string{"docker-compose.yml"}
}
} else {
context.ComposeFiles = c.GlobalStringSlice("file")
}

context.ProjectName = c.GlobalString("project-name")

if c.Command.Name == "logs" {
Expand Down
4 changes: 2 additions & 2 deletions example/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ import (
func main() {
project, err := docker.NewProject(&docker.Context{
Context: project.Context{
ComposeFile: "docker-compose.yml",
ProjectName: "yeah-compose",
ComposeFiles: []string{"docker-compose.yml"},
ProjectName: "yeah-compose",
},
})

Expand Down
10 changes: 10 additions & 0 deletions integration/assets/multiple/one.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
multiple:
image: tianon/true
environment:
- KEY1=VAL1
simple:
image: busybox:latest
command: top
another:
image: busybox:latest
command: top
8 changes: 8 additions & 0 deletions integration/assets/multiple/two.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
multiple:
image: busybox
command: echo two
environment:
- KEY2=VAL2
yetanother:
image: busybox:latest
command: top
51 changes: 51 additions & 0 deletions integration/create_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package integration
import (
"fmt"
"os"
"os/exec"

. "gopkg.in/check.v1"
)
Expand Down Expand Up @@ -156,3 +157,53 @@ func (s *RunSuite) TestFieldTypeConversions(c *C) {

os.Unsetenv("LIMIT")
}

func (s *RunSuite) TestMultipleComposeFiles(c *C) {
p := s.RandomProject()
cmd := exec.Command(s.command, "-f", "./assets/multiple/one.yml", "-f", "./assets/multiple/two.yml", "-p", p, "create")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Stdin = os.Stdin
err := cmd.Run()

c.Assert(err, IsNil)

containerNames := []string{"multiple", "simple", "another", "yetanother"}

for _, containerName := range containerNames {
name := fmt.Sprintf("%s_%s_1", p, containerName)
container := s.GetContainerByName(c, name)

c.Assert(container, NotNil)
}

name := fmt.Sprintf("%s_%s_1", p, "multiple")
container := s.GetContainerByName(c, name)

c.Assert(container.Config.Image, Equals, "busybox")
c.Assert(container.Config.Cmd, DeepEquals, []string{"echo", "two"})
c.Assert(container.Config.Env, DeepEquals, []string{"KEY1=VAL1", "KEY2=VAL2"})

p = s.RandomProject()
cmd = exec.Command(s.command, "-f", "./assets/multiple/two.yml", "-f", "./assets/multiple/one.yml", "-p", p, "create")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Stdin = os.Stdin
err = cmd.Run()

c.Assert(err, IsNil)

for _, containerName := range containerNames {
name := fmt.Sprintf("%s_%s_1", p, containerName)
container := s.GetContainerByName(c, name)

c.Assert(container, NotNil)
}

name = fmt.Sprintf("%s_%s_1", p, "multiple")
container = s.GetContainerByName(c, name)

c.Assert(container.Config.Image, Equals, "tianon/true")
c.Assert(container.Config.Cmd, DeepEquals, []string{"echo", "two"})
c.Assert(container.Config.Env, DeepEquals, []string{"KEY2=VAL2", "KEY1=VAL1"})
}
48 changes: 28 additions & 20 deletions project/context.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ type Context struct {
ForceRecreate bool
NoRecreate bool
Signal int
ComposeFile string
ComposeBytes []byte
ComposeFiles []string
ComposeBytes [][]byte
ProjectName string
isOpen bool
ServiceFactory ServiceFactory
Expand All @@ -36,32 +36,35 @@ type Context struct {
Project *Project
}

func (c *Context) readComposeFile() error {
func (c *Context) readComposeFiles() error {
Copy link
Copy Markdown
Contributor

@vdemeester vdemeester Dec 26, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This method feels a little bit complicated to me (too much complexity / indentation).

func (c *Context) readComposeFiles() error {
    if c.ComposeBytes != nil {
        return nil
    }

    logrus.Debugf("Opening compose files: %s", strings.Join(c.ComposeFiles, ","))

    // Handle STDIN (`-f -`)
    if len(c.ComposeFiles) == 1 && c.ComposeFiles[0] == "-" {
        composeBytes, err := ioutil.ReadAll(os.Stdin)
        if err != nil {
            logrus.Errorf("Failed to read compose file from stdin: %v", err)
            return err
        }
        c.ComposeBytes = [][]byte{composeBytes}
        return nil
    }

    for _, composeFile := range c.ComposeFiles {
        composeBytes, err := ioutil.ReadFile(composeFile)
        if err != nil {
            if os.IsNotExist(err) && !c.IgnoreMissingConfig {
                logrus.Errorf("Failed to find %s", composeFile)
                return err
            }
            logrus.Errorf("Failed to open %s", composeFile)
            return err
        }
        c.ComposeBytes = append(c.ComposeBytes, composeBytes)
    }

    return nil
}

or even :

func (c *Context) readComposeFiles() error {
    if c.ComposeBytes != nil {
        return nil
    }

    logrus.Debugf("Opening compose files: %s", strings.Join(c.ComposeFiles, ","))

    // Handle STDIN (`-f -`)
    if len(c.ComposeFiles) == 1 && c.ComposeFiles[0] == "-" {
        composeBytes, err := readComposeFile(os.Stdin)
        if err != nil {
            return err
        }
        c.ComposeBytes = [][]byte{composeBytes}
        return nil
    }

    for _, composeFile := range c.ComposeFiles {
        composeFileF, err := os.Open(composeFile)
        if os.IsNotExist(err) && !c.IgnoreMissingConfig {
            logrus.Errorf("Failed to find %s", composeFile)
            return err
        }
        composeBytes, err := readComposeFile(composeFileF)
        if err != nil {
            return err
        }
        c.ComposeBytes = append(c.ComposeBytes, composeBytes)
    }

    return nil
}

func readComposeFile(reader io.Reader) ([]byte, error) {
    composeBytes, err := ioutil.ReadAll(reader)
    if err != nil {
        logrus.Errorf("Failed to read compose file : %v", err)
        return nil, err
    }
    return composeBytes, nil
}

After looking at both, I prefer the first one 😺.

if c.ComposeBytes != nil {
return nil
}

logrus.Debugf("Opening compose file: %s", c.ComposeFile)
logrus.Debugf("Opening compose files: %s", strings.Join(c.ComposeFiles, ","))

if c.ComposeFile == "-" {
if len(c.ComposeFiles) == 1 && c.ComposeFiles[0] == "-" {
composeBytes, err := ioutil.ReadAll(os.Stdin)
if err != nil {
logrus.Errorf("Failed to read compose file from stdin: %v", err)
return err
}
c.ComposeBytes = composeBytes
} else if c.ComposeFile != "" {
if composeBytes, err := ioutil.ReadFile(c.ComposeFile); os.IsNotExist(err) {
if c.IgnoreMissingConfig {
return nil
c.ComposeBytes = [][]byte{composeBytes}
} else {
for _, composeFile := range c.ComposeFiles {
if composeFile != "" {
Copy link
Copy Markdown
Contributor

@vdemeester vdemeester Dec 26, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure we should do this check, docker-compose will fail if we manage to specify an empty compose file :

$ libcompose-cli -f ../ports-composefile/docker-compose.yml -f "" up -d
WARN[0000] Note: This is an experimental alternate implementation of the Compose CLI (https://github.com/docker/compose) 
INFO[0000] Project [ports-composefile]: Starting project  
INFO[0000] [0/1] [simple]: Starting                     
INFO[0000] [1/1] [simple]: Started                      

$ docker-compose -f ../ports-composefile/docker-compose.yml -f "" up -d
ERROR: .IOError: [Errno 21] Is a directory: u'./'

if composeBytes, err := ioutil.ReadFile(composeFile); os.IsNotExist(err) {
if !c.IgnoreMissingConfig {
logrus.Errorf("Failed to find %s", composeFile)
return err
}
} else if err != nil {
logrus.Errorf("Failed to open %s", composeFile)
return err
} else {
c.ComposeBytes = append(c.ComposeBytes, composeBytes)
}
}
logrus.Errorf("Failed to find %s", c.ComposeFile)
return err
} else if err != nil {
logrus.Errorf("Failed to open %s", c.ComposeFile)
return err
} else {
c.ComposeBytes = composeBytes
}
}

Expand Down Expand Up @@ -96,9 +99,14 @@ func (c *Context) lookupProjectName() (string, error) {
return envProject, nil
}

f, err := filepath.Abs(c.ComposeFile)
file := ""
Copy link
Copy Markdown
Contributor

@vdemeester vdemeester Dec 26, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would use file := "." to show that it takes the current directory by default — it's a nit, really 😝 .

if len(c.ComposeFiles) > 0 {
file = c.ComposeFiles[0]
}

f, err := filepath.Abs(file)
if err != nil {
logrus.Errorf("Failed to get absolute directory for: %s", c.ComposeFile)
logrus.Errorf("Failed to get absolute directory for: %s", file)
return "", err
}

Expand All @@ -123,7 +131,7 @@ func (c *Context) open() error {
return nil
}

if err := c.readComposeFile(); err != nil {
if err := c.readComposeFiles(); err != nil {
return err
}

Expand Down
29 changes: 23 additions & 6 deletions project/merge.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ var (
type rawService map[string]interface{}
type rawServiceMap map[string]rawService

func mergeProject(p *Project, bytes []byte) (map[string]*ServiceConfig, error) {
func mergeProject(p *Project, file string, bytes []byte) (map[string]*ServiceConfig, error) {
configs := make(map[string]*ServiceConfig)

datas := make(rawServiceMap)
Expand All @@ -44,20 +44,31 @@ func mergeProject(p *Project, bytes []byte) (map[string]*ServiceConfig, error) {
}

for name, data := range datas {
data, err := parse(p.context.ConfigLookup, p.context.EnvironmentLookup, p.File, data, datas)
data, err := parse(p.context.ConfigLookup, p.context.EnvironmentLookup, file, data, datas)

if err != nil {
logrus.Errorf("Failed to parse service %s: %v", name, err)
return nil, err
}

datas[name] = data
if _, ok := p.Configs[name]; ok {
Copy link
Copy Markdown
Contributor

@vdemeester vdemeester Dec 26, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

        if _, ok := p.Configs[name]; ok {
            var rawExistingService rawService
            if err := utils.Convert(p.Configs[name], &rawExistingService); err != nil {
                return nil, err
            }

            data = mergeConfig(rawExistingService, data)
        }

        datas[name] = data

var rawExistingService rawService
if err := utils.Convert(p.Configs[name], &rawExistingService); err != nil {
return nil, err
}

datas[name] = mergeConfig(rawExistingService, data)
} else {
datas[name] = data
}
}

if err := utils.Convert(datas, &configs); err != nil {
return nil, err
}

adjustValues(configs)

return configs, nil
}

Expand Down Expand Up @@ -237,6 +248,14 @@ func parse(configLookup ConfigLookup, environmentLookup EnvironmentLookup, inFil
}
}

baseService = mergeConfig(baseService, serviceData)

logrus.Debugf("Merged result %#v", baseService)

return baseService, nil
}

func mergeConfig(baseService, serviceData rawService) rawService {
for k, v := range serviceData {
// Image and build are mutually exclusive in merge
if k == "image" {
Expand All @@ -252,9 +271,7 @@ func parse(configLookup ConfigLookup, environmentLookup EnvironmentLookup, inFil
}
}

logrus.Debugf("Merged result %#v", baseService)

return baseService, nil
return baseService
}

func merge(existing, value interface{}) interface{} {
Expand Down
12 changes: 6 additions & 6 deletions project/merge_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ func TestExtendsInheritImage(t *testing.T) {
ConfigLookup: &NullLookup{},
})

config, err := mergeProject(p, []byte(`
config, err := mergeProject(p, "", []byte(`
parent:
image: foo
child:
Expand Down Expand Up @@ -47,7 +47,7 @@ func TestExtendsInheritBuild(t *testing.T) {
ConfigLookup: &NullLookup{},
})

config, err := mergeProject(p, []byte(`
config, err := mergeProject(p, "", []byte(`
parent:
build: .
child:
Expand Down Expand Up @@ -80,7 +80,7 @@ func TestExtendBuildOverImage(t *testing.T) {
ConfigLookup: &NullLookup{},
})

config, err := mergeProject(p, []byte(`
config, err := mergeProject(p, "", []byte(`
parent:
image: foo
child:
Expand Down Expand Up @@ -114,7 +114,7 @@ func TestExtendImageOverBuild(t *testing.T) {
ConfigLookup: &NullLookup{},
})

config, err := mergeProject(p, []byte(`
config, err := mergeProject(p, "", []byte(`
parent:
build: .
child:
Expand Down Expand Up @@ -152,7 +152,7 @@ func TestRestartNo(t *testing.T) {
ConfigLookup: &NullLookup{},
})

config, err := mergeProject(p, []byte(`
config, err := mergeProject(p, "", []byte(`
test:
restart: no
image: foo
Expand All @@ -174,7 +174,7 @@ func TestRestartAlways(t *testing.T) {
ConfigLookup: &NullLookup{},
})

config, err := mergeProject(p, []byte(`
config, err := mergeProject(p, "", []byte(`
test:
restart: always
image: foo
Expand Down
24 changes: 19 additions & 5 deletions project/project.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,14 +63,24 @@ func (p *Project) Parse() error {

p.Name = p.context.ProjectName

if p.context.ComposeFile == "-" {
p.File = "."
if len(p.context.ComposeFiles) == 1 && p.context.ComposeFiles[0] == "-" {
Copy link
Copy Markdown
Contributor

@vdemeester vdemeester Dec 26, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

    p.Files = p.context.ComposeFiles

    if len(p.Files) == 1 && p.Files[0] == "-" {
        p.Files = []string{"."}
    }

p.Files = []string{"."}
} else {
p.File = p.context.ComposeFile
p.Files = p.context.ComposeFiles
}

if p.context.ComposeBytes != nil {
return p.Load(p.context.ComposeBytes)
for i, composeBytes := range p.context.ComposeBytes {
if i < len(p.context.ComposeFiles) {
Copy link
Copy Markdown
Contributor

@vdemeester vdemeester Dec 26, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What about :

            file := ""
            if i < len(p.context.ComposeFiles) {
                file = p.Files[i]
            }
            if err := p.load(file, composeBytes); err != nil {
                return err
            }

But then, I would refactor the Load method to take a file string and not create load (if not used elsewhere).

if err := p.load(p.Files[i], composeBytes); err != nil {
return err
}
} else {
if err := p.Load(composeBytes); err != nil {
return err
}
}
}
}

return nil
Expand Down Expand Up @@ -123,8 +133,12 @@ func (p *Project) AddConfig(name string, config *ServiceConfig) error {
// Load loads the specified byte array (the composefile content) and adds the
// service configuration to the project.
func (p *Project) Load(bytes []byte) error {
return p.load("", bytes)
}

func (p *Project) load(file string, bytes []byte) error {
configs := make(map[string]*ServiceConfig)
configs, err := mergeProject(p, bytes)
configs, err := mergeProject(p, file, bytes)
if err != nil {
log.Errorf("Could not parse config for project %s : %v", p.Name, err)
return err
Expand Down
Loading