Introduce new config format with volume support and versioning

compose/cli/command.py

@@ -85,7 +85,7 @@ def get_project(base_dir, config_path=None, project_name=None, verbose=False,
 
     api_version = '1.21' if use_networking else None
     try:
-        return Project.from_dicts(
+        return Project.from_config(
             get_project_name(config_details.working_dir, project_name),
             config.load(config_details),
             get_client(verbose=verbose, version=api_version),

compose/cli/docker_client.py

@@ -9,7 +9,7 @@
 log = logging.getLogger(__name__)
 
 
-DEFAULT_API_VERSION = '1.19'
+DEFAULT_API_VERSION = '1.21'
 
 
 def docker_client(version=None):

compose/config/config.py

@@ -100,6 +100,17 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')):
     """
 
 
+class Config(namedtuple('_Config', 'version services volumes')):
+    """
+    :param version: configuration version
+    :type  version: int
+    :param services: List of service description dictionaries
+    :type  services: :class:`list`
+    :param volumes: List of volume description dictionaries
+    :type  volumes: :class:`list`
+    """
+
+
 def find(base_dir, filenames):
     if filenames == ['-']:
         return ConfigDetails(
@@ -165,17 +176,76 @@ def find_candidates_in_parent_dirs(filenames, path):
     return (candidates, path)
 
 
+def get_config_version(config_details):
+    def get_version(config):
+        validate_top_level_object(config)
+        return config.get('version')
+
+    main_file = config_details.config_files[0]
+    version = get_version(main_file.config)
+    for next_file in config_details.config_files[1:]:
+        next_file_version = get_version(main_file.config)
+        if version != next_file_version:
+            raise ConfigurationError(
+                'Version mismatch: main file {0} specifies version {1} but extension file {2} uses version {3}'.format(
+                    main_file.filename, version, next_file.filename, next_file_version
+                )
+            )
+    return version, main_file.filename
+
+
+def pre_process_config(config):
+    """
+    Pre validation checks and processing of the config file to interpolate env
+    vars returning a config dict ready to be tested against the schema.
+    """
+    validate_top_level_object(config)
+    return interpolate_environment_variables(config)
+
+
 def load(config_details):
-    """Load the configuration from a working directory and a list of
-    configuration files.  Files are loaded in order, and merged on top
-    of each other to create the final configuration.
+    version, filename = get_config_version(config_details)
+    if not version or isinstance(version, dict):
+        service_dicts = [
+            config_file.config for config_file in config_details.config_files
+        ]
+        return Config(
+            version,
+            load_services(config_details.working_dir, filename, service_dicts),
+            {}
+        )
+    elif version == 2:
+        return load_v2(config_details, filename)
+
+    raise ConfigurationError('Invalid config version provided: {0}'.format(version))
 
-    Return a fully interpolated, extended and validated configuration.
+
+def load_v2(config_details, filename):
+    service_dicts = [
+        config_file.config.get('services', {}) for config_file in config_details.config_files
+    ]
+    volumes = {}
+    for config_file in config_details.config_files:
+        for name, volume_config in config_file.config.get('volumes', {}).items():
+            volumes.update({name: volume_config})
+    return Config(
+        2,
+        load_services(config_details.working_dir, filename, service_dicts),
+        volumes
+    )
+
+
+def load_services(working_dir, filename, service_configs):
+    """Load the configuration from a working directory, filename and a list of
+    services dicts.  Dicts are loaded in order, and merged on top of each
+    other to create the final configuration.
+
+    Return a fully interpolated, extended and validated service configuration.
     """
 
     def build_service(filename, service_name, service_dict):
         loader = ServiceLoader(
-            config_details.working_dir,
+            working_dir,
             filename,
             service_name,
             service_dict)
@@ -184,8 +254,8 @@ def build_service(filename, service_name, service_dict):
         return service_dict
 
     def load_file(filename, config):
-        processed_config = interpolate_environment_variables(config)
-        validate_against_fields_schema(processed_config)
+        processed_config = pre_process_config(config)
+        validate_against_fields_schema(processed_config, None)
         return [
             build_service(filename, name, service_config)
             for name, service_config in processed_config.items()
@@ -195,21 +265,17 @@ def merge_services(base, override):
         all_service_names = set(base) | set(override)
         return {
             name: merge_service_dicts_from_files(
-                base.get(name, {}),
-                override.get(name, {}))
+                base.get(name, {}), override.get(name, {})
+            )
             for name in all_service_names
         }
 
-    config_file = config_details.config_files[0]
-    validate_top_level_object(config_file.config)
-    for next_file in config_details.config_files[1:]:
-        validate_top_level_object(next_file.config)
-
-        config_file = ConfigFile(
-            config_file.filename,
-            merge_services(config_file.config, next_file.config))
+    service_config = service_configs[0]
+    for next_config in service_configs[1:]:
+        validate_top_level_object(next_config)
+        service_config = merge_services(service_config, next_config)
 
-    return load_file(config_file.filename, config_file.config)
+    return load_file(filename, service_config)
 
 
 class ServiceLoader(object):

compose/config/fields_schema_v2.json

@@ -0,0 +1,43 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+
+  "type": "object",
+  "properties": {
+    "version": {
+      "enum": [2]
+    },
+    "services": {
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "fields_schema.json#/definitions/service"
+        }
+      }
+    },
+    "volumes": {
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/volume"
+        }
+      }
+    }
+  },
+
+  "definitions": {
+    "volume": {
+      "type": "object",
+      "properties": {
+        "driver": {"type": "string"},
+        "driver_opts": {
+          "type": "object",
+          "patternProperties": {
+            "^.+$": {"type": ["boolean", "string", "number"]}
+          },
+          "additionalProperties": false
+        }
+      }
+    }
+  },
+  "additionalProperties": false
+}

compose/config/validation.py

@@ -287,8 +287,10 @@ def _parse_oneof_validator(error):
     return "\n".join(root_msgs + invalid_keys + required + type_errors + other_errors)
 
 
-def validate_against_fields_schema(config):
+def validate_against_fields_schema(config, version=None):
     schema_filename = "fields_schema.json"
+    if version:
+        schema_filename = "fields_schema_v{0}.json".format(version)
     format_checkers = ["ports", "environment"]
     return _validate_against_schema(config, schema_filename, format_checkers)
 

compose/container.py

@@ -171,6 +171,12 @@ def get_local_port(self, port, protocol='tcp'):
         port = self.ports.get("%s/%s" % (port, protocol))
         return "{HostIp}:{HostPort}".format(**port[0]) if port else None
 
+    def get_mount(self, mount_dest):
+        for mount in self.get('Mounts'):
+            if mount['Destination'] == mount_dest:
+                return mount
+        return None
+
     def start(self, **options):
         return self.client.start(self.id, **options)
 

compose/project.py

@@ -23,6 +23,7 @@
 from .service import ServiceNet
 from .service import VolumeFromSpec
 from .utils import parallel_execute
+from .volume import Volume
 
 
 log = logging.getLogger(__name__)
@@ -78,10 +79,11 @@ class Project(object):
     """
     A collection of services.
     """
-    def __init__(self, name, services, client, use_networking=False, network_driver=None):
+    def __init__(self, name, services, client, volumes=None, use_networking=False, network_driver=None):
         self.name = name
         self.services = services
         self.client = client
+        self.volumes = volumes or []
         self.use_networking = use_networking
         self.network_driver = network_driver
 
@@ -92,16 +94,16 @@ def labels(self, one_off=False):
         ]
 
     @classmethod
-    def from_dicts(cls, name, service_dicts, client, use_networking=False, network_driver=None):
+    def from_config(cls, name, config_data, client, use_networking=False, network_driver=None):
         """
-        Construct a ServiceCollection from a list of dicts representing services.
+        Construct a Project from a config.Config object.
         """
         project = cls(name, [], client, use_networking=use_networking, network_driver=network_driver)
 
         if use_networking:
-            remove_links(service_dicts)
+            remove_links(config_data.services)
 
-        for service_dict in sort_service_dicts(service_dicts):
+        for service_dict in sort_service_dicts(config_data.services):
             links = project.get_links(service_dict)
             volumes_from = project.get_volumes_from(service_dict)
             net = project.get_net(service_dict)
@@ -115,6 +117,15 @@ def from_dicts(cls, name, service_dicts, client, use_networking=False, network_d
                     net=net,
                     volumes_from=volumes_from,
                     **service_dict))
+
+        for vol_name, data in config_data.volumes.items():
+            project.volumes.append(
+                Volume(
+                    client=client, project=name, name=vol_name,
+                    driver=data.get('driver'), driver_opts=data.get('driver_opts')
+                )
+            )
+
         return project
 
     @property
@@ -274,6 +285,15 @@ def remove_stopped(self, service_names=None, **options):
             msg="Removing"
         )
 
+    def initialize_volumes(self):
+        try:
+            for volume in self.volumes:
+                volume.create()
+        except NotFound:
+            raise ConfigurationError(
+                "Volume %s specifies nonexistent driver %s" % (volume.name, volume.driver)
+            )
+
     def restart(self, service_names=None, **options):
         for service in self.get_services(service_names):
             service.restart(**options)
@@ -293,6 +313,8 @@ def up(self,
            timeout=DEFAULT_TIMEOUT,
            detached=False):
 
+        self.initialize_volumes()
+
         services = self.get_services(service_names, include_deps=start_deps)
 
         for service in services:

compose/service.py

@@ -930,7 +930,7 @@ def get_container_data_volumes(container, volumes_option):
     volumes = []
 
     volumes_option = volumes_option or []
-    container_volumes = container.get('Volumes') or {}
+    container_volumes = container.get('Mounts') or []
     image_volumes = container.image_config['ContainerConfig'].get('Volumes') or {}
 
     for volume in set(volumes_option + list(image_volumes)):
@@ -939,7 +939,11 @@ def get_container_data_volumes(container, volumes_option):
         if volume.external:
             continue
 
-        volume_path = container_volumes.get(volume.internal)
+        volume_path = None
+        for volume_config in container_volumes:
+            if volume_config['Destination'] == volume.internal:
+                volume_path = volume_config['Source']
+                break
         # New volume, doesn't exist in the old container
         if not volume_path:
             continue

compose/volume.py

@@ -0,0 +1,19 @@
+from __future__ import unicode_literals
+
+
+class Volume(object):
+    def __init__(self, client, project, name, driver=None, driver_opts=None):
+        self.client = client
+        self.project = project
+        self.name = name
+        self.driver = driver
+        self.driver_opts = driver_opts
+
+    def create(self):
+        return self.client.create_volume(self.name, self.driver, self.driver_opts)
+
+    def remove(self):
+        return self.client.remove_volume(self.name)
+
+    def inspect(self):
+        return self.client.inspect_volume(self.name)