diff --git a/README.md b/README.md index d1a8bacff..05e5d6102 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ import ( func main() { project, err := docker.NewProject(&docker.Context{ Context: project.Context{ - ComposeFile: "docker-compose.yml", - ProjectName: "my-compose", + ComposeFiles: []string{"docker-compose.yml"}, + ProjectName: "my-compose", }, }) diff --git a/cli/command/command.go b/cli/command/command.go index d999abd34..6fbd7c16d 100644 --- a/cli/command/command.go +++ b/cli/command/command.go @@ -1,6 +1,8 @@ package command import ( + "os" + "github.com/codegangsta/cli" "github.com/docker/libcompose/cli/app" "github.com/docker/libcompose/project" @@ -213,10 +215,10 @@ func CommonFlags() []cli.Flag { cli.BoolFlag{ Name: "verbose,debug", }, - cli.StringFlag{ + cli.StringSliceFlag{ Name: "file,f", - Usage: "Specify an alternate compose file (default: docker-compose.yml)", - Value: "docker-compose.yml", + Usage: "Specify one or more alternate compose files (default: docker-compose.yml)", + Value: &cli.StringSlice{}, EnvVar: "COMPOSE_FILE", }, cli.StringFlag{ @@ -228,7 +230,16 @@ func CommonFlags() []cli.Flag { // Populate updates the specified project context based on command line arguments and subcommands. func Populate(context *project.Context, c *cli.Context) { - context.ComposeFile = c.GlobalString("file") + if len(c.GlobalStringSlice("file")) == 0 { + if _, err := os.Stat("docker-compose.override.yml"); err == nil { + context.ComposeFiles = []string{"docker-compose.yml", "docker-compose.override.yml"} + } else { + context.ComposeFiles = []string{"docker-compose.yml"} + } + } else { + context.ComposeFiles = c.GlobalStringSlice("file") + } + context.ProjectName = c.GlobalString("project-name") if c.Command.Name == "logs" { diff --git a/example/main.go b/example/main.go index a39a8bd9a..dca7a1119 100644 --- a/example/main.go +++ b/example/main.go @@ -10,8 +10,8 @@ import ( func main() { project, err := docker.NewProject(&docker.Context{ Context: project.Context{ - ComposeFile: "docker-compose.yml", - ProjectName: "yeah-compose", + ComposeFiles: []string{"docker-compose.yml"}, + ProjectName: "yeah-compose", }, }) diff --git a/integration/assets/multiple/one.yml b/integration/assets/multiple/one.yml new file mode 100644 index 000000000..78381b6ef --- /dev/null +++ b/integration/assets/multiple/one.yml @@ -0,0 +1,10 @@ +multiple: + image: tianon/true + environment: + - KEY1=VAL1 +simple: + image: busybox:latest + command: top +another: + image: busybox:latest + command: top diff --git a/integration/assets/multiple/two.yml b/integration/assets/multiple/two.yml new file mode 100644 index 000000000..db82fa393 --- /dev/null +++ b/integration/assets/multiple/two.yml @@ -0,0 +1,8 @@ +multiple: + image: busybox + command: echo two + environment: + - KEY2=VAL2 +yetanother: + image: busybox:latest + command: top diff --git a/integration/create_test.go b/integration/create_test.go index c096716dd..808a6ac0d 100644 --- a/integration/create_test.go +++ b/integration/create_test.go @@ -3,6 +3,7 @@ package integration import ( "fmt" "os" + "os/exec" . "gopkg.in/check.v1" ) @@ -156,3 +157,53 @@ func (s *RunSuite) TestFieldTypeConversions(c *C) { os.Unsetenv("LIMIT") } + +func (s *RunSuite) TestMultipleComposeFiles(c *C) { + p := s.RandomProject() + cmd := exec.Command(s.command, "-f", "./assets/multiple/one.yml", "-f", "./assets/multiple/two.yml", "-p", p, "create") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdin + err := cmd.Run() + + c.Assert(err, IsNil) + + containerNames := []string{"multiple", "simple", "another", "yetanother"} + + for _, containerName := range containerNames { + name := fmt.Sprintf("%s_%s_1", p, containerName) + container := s.GetContainerByName(c, name) + + c.Assert(container, NotNil) + } + + name := fmt.Sprintf("%s_%s_1", p, "multiple") + container := s.GetContainerByName(c, name) + + c.Assert(container.Config.Image, Equals, "busybox") + c.Assert(container.Config.Cmd, DeepEquals, []string{"echo", "two"}) + c.Assert(container.Config.Env, DeepEquals, []string{"KEY1=VAL1", "KEY2=VAL2"}) + + p = s.RandomProject() + cmd = exec.Command(s.command, "-f", "./assets/multiple/two.yml", "-f", "./assets/multiple/one.yml", "-p", p, "create") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdin + err = cmd.Run() + + c.Assert(err, IsNil) + + for _, containerName := range containerNames { + name := fmt.Sprintf("%s_%s_1", p, containerName) + container := s.GetContainerByName(c, name) + + c.Assert(container, NotNil) + } + + name = fmt.Sprintf("%s_%s_1", p, "multiple") + container = s.GetContainerByName(c, name) + + c.Assert(container.Config.Image, Equals, "tianon/true") + c.Assert(container.Config.Cmd, DeepEquals, []string{"echo", "two"}) + c.Assert(container.Config.Env, DeepEquals, []string{"KEY2=VAL2", "KEY1=VAL1"}) +} diff --git a/project/context.go b/project/context.go index ba82c1d7b..d9b5e88c1 100644 --- a/project/context.go +++ b/project/context.go @@ -24,8 +24,8 @@ type Context struct { ForceRecreate bool NoRecreate bool Signal int - ComposeFile string - ComposeBytes []byte + ComposeFiles []string + ComposeBytes [][]byte ProjectName string isOpen bool ServiceFactory ServiceFactory @@ -36,32 +36,35 @@ type Context struct { Project *Project } -func (c *Context) readComposeFile() error { +func (c *Context) readComposeFiles() error { if c.ComposeBytes != nil { return nil } - logrus.Debugf("Opening compose file: %s", c.ComposeFile) + logrus.Debugf("Opening compose files: %s", strings.Join(c.ComposeFiles, ",")) - if c.ComposeFile == "-" { + if len(c.ComposeFiles) == 1 && c.ComposeFiles[0] == "-" { composeBytes, err := ioutil.ReadAll(os.Stdin) if err != nil { logrus.Errorf("Failed to read compose file from stdin: %v", err) return err } - c.ComposeBytes = composeBytes - } else if c.ComposeFile != "" { - if composeBytes, err := ioutil.ReadFile(c.ComposeFile); os.IsNotExist(err) { - if c.IgnoreMissingConfig { - return nil + c.ComposeBytes = [][]byte{composeBytes} + } else { + for _, composeFile := range c.ComposeFiles { + if composeFile != "" { + if composeBytes, err := ioutil.ReadFile(composeFile); os.IsNotExist(err) { + if !c.IgnoreMissingConfig { + logrus.Errorf("Failed to find %s", composeFile) + return err + } + } else if err != nil { + logrus.Errorf("Failed to open %s", composeFile) + return err + } else { + c.ComposeBytes = append(c.ComposeBytes, composeBytes) + } } - logrus.Errorf("Failed to find %s", c.ComposeFile) - return err - } else if err != nil { - logrus.Errorf("Failed to open %s", c.ComposeFile) - return err - } else { - c.ComposeBytes = composeBytes } } @@ -96,9 +99,14 @@ func (c *Context) lookupProjectName() (string, error) { return envProject, nil } - f, err := filepath.Abs(c.ComposeFile) + file := "" + if len(c.ComposeFiles) > 0 { + file = c.ComposeFiles[0] + } + + f, err := filepath.Abs(file) if err != nil { - logrus.Errorf("Failed to get absolute directory for: %s", c.ComposeFile) + logrus.Errorf("Failed to get absolute directory for: %s", file) return "", err } @@ -123,7 +131,7 @@ func (c *Context) open() error { return nil } - if err := c.readComposeFile(); err != nil { + if err := c.readComposeFiles(); err != nil { return err } diff --git a/project/merge.go b/project/merge.go index 9bf5c420f..df3752268 100644 --- a/project/merge.go +++ b/project/merge.go @@ -31,7 +31,7 @@ var ( type rawService map[string]interface{} type rawServiceMap map[string]rawService -func mergeProject(p *Project, bytes []byte) (map[string]*ServiceConfig, error) { +func mergeProject(p *Project, file string, bytes []byte) (map[string]*ServiceConfig, error) { configs := make(map[string]*ServiceConfig) datas := make(rawServiceMap) @@ -44,13 +44,23 @@ func mergeProject(p *Project, bytes []byte) (map[string]*ServiceConfig, error) { } for name, data := range datas { - data, err := parse(p.context.ConfigLookup, p.context.EnvironmentLookup, p.File, data, datas) + data, err := parse(p.context.ConfigLookup, p.context.EnvironmentLookup, file, data, datas) + if err != nil { logrus.Errorf("Failed to parse service %s: %v", name, err) return nil, err } - datas[name] = data + if _, ok := p.Configs[name]; ok { + var rawExistingService rawService + if err := utils.Convert(p.Configs[name], &rawExistingService); err != nil { + return nil, err + } + + datas[name] = mergeConfig(rawExistingService, data) + } else { + datas[name] = data + } } if err := utils.Convert(datas, &configs); err != nil { @@ -58,6 +68,7 @@ func mergeProject(p *Project, bytes []byte) (map[string]*ServiceConfig, error) { } adjustValues(configs) + return configs, nil } @@ -237,6 +248,14 @@ func parse(configLookup ConfigLookup, environmentLookup EnvironmentLookup, inFil } } + baseService = mergeConfig(baseService, serviceData) + + logrus.Debugf("Merged result %#v", baseService) + + return baseService, nil +} + +func mergeConfig(baseService, serviceData rawService) rawService { for k, v := range serviceData { // Image and build are mutually exclusive in merge if k == "image" { @@ -252,9 +271,7 @@ func parse(configLookup ConfigLookup, environmentLookup EnvironmentLookup, inFil } } - logrus.Debugf("Merged result %#v", baseService) - - return baseService, nil + return baseService } func merge(existing, value interface{}) interface{} { diff --git a/project/merge_test.go b/project/merge_test.go index 3ffe656a5..7084f8aa8 100644 --- a/project/merge_test.go +++ b/project/merge_test.go @@ -14,7 +14,7 @@ func TestExtendsInheritImage(t *testing.T) { ConfigLookup: &NullLookup{}, }) - config, err := mergeProject(p, []byte(` + config, err := mergeProject(p, "", []byte(` parent: image: foo child: @@ -47,7 +47,7 @@ func TestExtendsInheritBuild(t *testing.T) { ConfigLookup: &NullLookup{}, }) - config, err := mergeProject(p, []byte(` + config, err := mergeProject(p, "", []byte(` parent: build: . child: @@ -80,7 +80,7 @@ func TestExtendBuildOverImage(t *testing.T) { ConfigLookup: &NullLookup{}, }) - config, err := mergeProject(p, []byte(` + config, err := mergeProject(p, "", []byte(` parent: image: foo child: @@ -114,7 +114,7 @@ func TestExtendImageOverBuild(t *testing.T) { ConfigLookup: &NullLookup{}, }) - config, err := mergeProject(p, []byte(` + config, err := mergeProject(p, "", []byte(` parent: build: . child: @@ -152,7 +152,7 @@ func TestRestartNo(t *testing.T) { ConfigLookup: &NullLookup{}, }) - config, err := mergeProject(p, []byte(` + config, err := mergeProject(p, "", []byte(` test: restart: no image: foo @@ -174,7 +174,7 @@ func TestRestartAlways(t *testing.T) { ConfigLookup: &NullLookup{}, }) - config, err := mergeProject(p, []byte(` + config, err := mergeProject(p, "", []byte(` test: restart: always image: foo diff --git a/project/project.go b/project/project.go index 870b791d4..031ded1f5 100644 --- a/project/project.go +++ b/project/project.go @@ -63,14 +63,24 @@ func (p *Project) Parse() error { p.Name = p.context.ProjectName - if p.context.ComposeFile == "-" { - p.File = "." + if len(p.context.ComposeFiles) == 1 && p.context.ComposeFiles[0] == "-" { + p.Files = []string{"."} } else { - p.File = p.context.ComposeFile + p.Files = p.context.ComposeFiles } if p.context.ComposeBytes != nil { - return p.Load(p.context.ComposeBytes) + for i, composeBytes := range p.context.ComposeBytes { + if i < len(p.context.ComposeFiles) { + if err := p.load(p.Files[i], composeBytes); err != nil { + return err + } + } else { + if err := p.Load(composeBytes); err != nil { + return err + } + } + } } return nil @@ -123,8 +133,12 @@ func (p *Project) AddConfig(name string, config *ServiceConfig) error { // Load loads the specified byte array (the composefile content) and adds the // service configuration to the project. func (p *Project) Load(bytes []byte) error { + return p.load("", bytes) +} + +func (p *Project) load(file string, bytes []byte) error { configs := make(map[string]*ServiceConfig) - configs, err := mergeProject(p, bytes) + configs, err := mergeProject(p, file, bytes) if err != nil { log.Errorf("Could not parse config for project %s : %v", p.Name, err) return err diff --git a/project/project_test.go b/project/project_test.go index deed9f697..6bfc6f03c 100644 --- a/project/project_test.go +++ b/project/project_test.go @@ -5,6 +5,8 @@ import ( "reflect" "strings" "testing" + + "github.com/stretchr/testify/assert" ) type TestServiceFactory struct { @@ -87,7 +89,9 @@ func TestEventEquality(t *testing.T) { func TestParseWithBadContent(t *testing.T) { p := NewProject(&Context{ - ComposeBytes: []byte("garbage"), + ComposeBytes: [][]byte{ + []byte("garbage"), + }, }) err := p.Parse() @@ -102,7 +106,9 @@ func TestParseWithBadContent(t *testing.T) { func TestParseWithGoodContent(t *testing.T) { p := NewProject(&Context{ - ComposeBytes: []byte("not-garbage:\n image: foo"), + ComposeBytes: [][]byte{ + []byte("not-garbage:\n image: foo"), + }, }) err := p.Parse() @@ -146,3 +152,61 @@ func TestEnvironmentResolve(t *testing.T) { t.Fatal("Invalid environment", service.Config().Environment.Slice()) } } + +func TestParseWithMultipleComposeFiles(t *testing.T) { + configOne := []byte(` + multiple: + image: tianon/true + ports: + - 8000`) + + configTwo := []byte(` + multiple: + image: busybox + name: multi + ports: + - 9000`) + + configThree := []byte(` + multiple: + mem_limit: 40000000 + ports: + - 10000`) + + p := NewProject(&Context{ + ComposeBytes: [][]byte{configOne, configTwo}, + }) + + err := p.Parse() + + assert.Nil(t, err) + + assert.Equal(t, "busybox", p.Configs["multiple"].Image) + assert.Equal(t, "multi", p.Configs["multiple"].Name) + assert.Equal(t, []string{"8000", "9000"}, p.Configs["multiple"].Ports) + + p = NewProject(&Context{ + ComposeBytes: [][]byte{configTwo, configOne}, + }) + + err = p.Parse() + + assert.Nil(t, err) + + assert.Equal(t, "tianon/true", p.Configs["multiple"].Image) + assert.Equal(t, "multi", p.Configs["multiple"].Name) + assert.Equal(t, []string{"9000", "8000"}, p.Configs["multiple"].Ports) + + p = NewProject(&Context{ + ComposeBytes: [][]byte{configOne, configTwo, configThree}, + }) + + err = p.Parse() + + assert.Nil(t, err) + + assert.Equal(t, "busybox", p.Configs["multiple"].Image) + assert.Equal(t, "multi", p.Configs["multiple"].Name) + assert.Equal(t, []string{"8000", "9000", "10000"}, p.Configs["multiple"].Ports) + assert.Equal(t, int64(40000000), p.Configs["multiple"].MemLimit) +} diff --git a/project/types.go b/project/types.go index c0b12915d..354e1240c 100644 --- a/project/types.go +++ b/project/types.go @@ -219,7 +219,7 @@ type ConfigLookup interface { type Project struct { Name string Configs map[string]*ServiceConfig - File string + Files []string ReloadCallback func() error context *Context reload []string