Add support for declaring named volumes in compose files

compose/cli/command.py

@@ -80,12 +80,13 @@ def get_project(base_dir, config_path=None, project_name=None, verbose=False,
     config_details = config.find(base_dir, config_path)
 
     api_version = '1.21' if use_networking else None
-    return Project.from_dicts(
+    return Project.from_config(
         get_project_name(config_details.working_dir, project_name),
         config.load(config_details),
         get_client(verbose=verbose, version=api_version),
         use_networking=use_networking,
-        network_driver=network_driver)
+        network_driver=network_driver
+    )
 
 
 def get_project_name(working_dir, project_name=None):

compose/cli/docker_client.py

@@ -8,8 +8,7 @@
 
 log = logging.getLogger(__name__)
 
-
-DEFAULT_API_VERSION = '1.19'
+DEFAULT_API_VERSION = '1.21'
 
 
 def docker_client(version=None):

compose/cli/main.py

@@ -211,11 +211,11 @@ def config(self, config_options, options):
             return
 
         if options['--services']:
-            print('\n'.join(service['name'] for service in compose_config))
+            print('\n'.join(service['name'] for service in compose_config.services))
             return
 
         compose_config = dict(
-            (service.pop('name'), service) for service in compose_config)
+            (service.pop('name'), service) for service in compose_config.services)
         print(yaml.dump(
             compose_config,
             default_flow_style=False,

compose/config/config.py

@@ -10,6 +10,7 @@
 import six
 import yaml
 
+from ..const import COMPOSEFILE_VERSIONS
 from .errors import CircularReference
 from .errors import ComposeFileNotFound
 from .errors import ConfigurationError
@@ -24,6 +25,7 @@
 from .validation import validate_against_service_schema
 from .validation import validate_extends_file_path
 from .validation import validate_top_level_object
+from .validation import validate_top_level_service_objects
 
 
 DOCKER_CONFIG_KEYS = [
@@ -116,6 +118,20 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')):
     def from_filename(cls, filename):
         return cls(filename, load_yaml(filename))
 
+    def get_service_dicts(self, version):
+        return self.config if version == 1 else self.config.get('services', {})
+
+
+class Config(namedtuple('_Config', 'version services volumes')):
+    """
+    :param version: configuration version
+    :type  version: int
+    :param services: List of service description dictionaries
+    :type  services: :class:`list`
+    :param volumes: List of volume description dictionaries
+    :type  volumes: :class:`list`
+    """
+
 
 class ServiceConfig(namedtuple('_ServiceConfig', 'working_dir filename name config')):
 
@@ -148,6 +164,34 @@ def find(base_dir, filenames):
         [ConfigFile.from_filename(f) for f in filenames])
 
 
+def get_config_version(config_details):
+    def get_version(config):
+        if config.config is None:
+            return 1
+        version = config.config.get('version', 1)
+        if isinstance(version, dict):
+            # in that case 'version' is probably a service name, so assume
+            # this is a legacy (version=1) file
+            version = 1
+        return version
+
+    main_file = config_details.config_files[0]
+    validate_top_level_object(main_file)
+    version = get_version(main_file)
+    for next_file in config_details.config_files[1:]:
+        validate_top_level_object(next_file)
+        next_file_version = get_version(next_file)
+
+        if version != next_file_version and next_file_version is not None:
+            raise ConfigurationError(
+                "Version mismatch: main file {0} specifies version {1} but "
+                "extension file {2} uses version {3}".format(
+                    main_file.filename, version, next_file.filename, next_file_version
+                )
+            )
+    return version
+
+
 def get_default_config_files(base_dir):
     (candidates, path) = find_candidates_in_parent_dirs(SUPPORTED_FILENAMES, base_dir)
 
@@ -194,14 +238,52 @@ def load(config_details):
 
     Return a fully interpolated, extended and validated configuration.
     """
+    version = get_config_version(config_details)
+    if version not in COMPOSEFILE_VERSIONS:
+        raise ConfigurationError('Invalid config version provided: {0}'.format(version))
+
+    processed_files = []
+    for config_file in config_details.config_files:
+        processed_files.append(
+            process_config_file(config_file, version=version)
+        )
+    config_details = config_details._replace(config_files=processed_files)
+
+    if version == 1:
+        service_dicts = load_services(
+            config_details.working_dir, config_details.config_files,
+            version
+        )
+        volumes = {}
+    elif version == 2:
+        config_files = [
+            ConfigFile(f.filename, f.config.get('services', {}))
+            for f in config_details.config_files
+        ]
+        service_dicts = load_services(
+            config_details.working_dir, config_files, version
+        )
+        volumes = load_volumes(config_details.config_files)
+
+    return Config(version, service_dicts, volumes)
+
+
+def load_volumes(config_files):
+    volumes = {}
+    for config_file in config_files:
+        for name, volume_config in config_file.config.get('volumes', {}).items():
+            volumes.update({name: volume_config})
+    return volumes
 
+
+def load_services(working_dir, config_files, version):
     def build_service(filename, service_name, service_dict):
         service_config = ServiceConfig.with_abs_paths(
-            config_details.working_dir,
+            working_dir,
             filename,
             service_name,
             service_dict)
-        resolver = ServiceExtendsResolver(service_config)
+        resolver = ServiceExtendsResolver(service_config, version)
         service_dict = process_service(resolver.run())
 
         # TODO: move to validate_service()
@@ -227,20 +309,28 @@ def merge_services(base, override):
             for name in all_service_names
         }
 
-    config_file = process_config_file(config_details.config_files[0])
-    for next_file in config_details.config_files[1:]:
-        next_file = process_config_file(next_file)
-
+    config_file = config_files[0]
+    for next_file in config_files[1:]:
         config = merge_services(config_file.config, next_file.config)
         config_file = config_file._replace(config=config)
 
     return build_services(config_file)
 
 
-def process_config_file(config_file, service_name=None):
-    validate_top_level_object(config_file)
-    processed_config = interpolate_environment_variables(config_file.config)
-    validate_against_fields_schema(processed_config, config_file.filename)
+def process_config_file(config_file, version, service_name=None):
+    service_dicts = config_file.get_service_dicts(version)
+    validate_top_level_service_objects(
+        config_file.filename, service_dicts
+    )
+    interpolated_config = interpolate_environment_variables(service_dicts)
+    if version == 2:
+        processed_config = dict(config_file.config)
+        processed_config.update({'services': interpolated_config})
+    if version == 1:
+        processed_config = interpolated_config
+    validate_against_fields_schema(
+        processed_config, config_file.filename, version
+    )
 
     if service_name and service_name not in processed_config:
         raise ConfigurationError(
@@ -251,10 +341,11 @@ def process_config_file(config_file, service_name=None):
 
 
 class ServiceExtendsResolver(object):
-    def __init__(self, service_config, already_seen=None):
+    def __init__(self, service_config, version, already_seen=None):
         self.service_config = service_config
         self.working_dir = service_config.working_dir
         self.already_seen = already_seen or []
+        self.version = version
 
     @property
     def signature(self):
@@ -283,7 +374,8 @@ def validate_and_construct_extends(self):
 
         extended_file = process_config_file(
             ConfigFile.from_filename(config_path),
-            service_name=service_name)
+            version=self.version, service_name=service_name
+        )
         service_config = extended_file.config[service_name]
         return config_path, service_config, service_name
 
@@ -294,6 +386,7 @@ def resolve_extends(self, extended_config_path, service_dict, service_name):
                 extended_config_path,
                 service_name,
                 service_dict),
+            self.version,
             already_seen=self.already_seen + [self.signature])
 
         service_config = resolver.run()

compose/config/fields_schema.json → compose/config/fields_schema_v1.json

@@ -2,7 +2,7 @@
   "$schema": "http://json-schema.org/draft-04/schema#",
 
   "type": "object",
-  "id": "fields_schema.json",
+  "id": "fields_schema_v1.json",
 
   "patternProperties": {
     "^[a-zA-Z0-9._-]+$": {

compose/config/fields_schema_v2.json

@@ -0,0 +1,49 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "id": "fields_schema_v2.json",
+
+  "properties": {
+    "version": {
+      "enum": [2]
+    },
+    "services": {
+      "id": "#/properties/services",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "fields_schema_v1.json#/definitions/service"
+        }
+      },
+      "additionalProperties": false
+    },
+    "volumes": {
+      "id": "#/properties/volumes",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/volume"
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+
+  "definitions": {
+    "volume": {
+      "id": "#/definitions/volume",
+      "type": "object",
+      "properties": {
+        "driver": {"type": "string"},
+        "driver_opts": {
+          "type": "object",
+          "patternProperties": {
+            "^.+$": {"type": ["string", "number"]}
+          },
+          "additionalProperties": false
+        }
+      }
+    }
+  },
+  "additionalProperties": false
+}

compose/config/interpolation.py

@@ -8,12 +8,12 @@
 log = logging.getLogger(__name__)
 
 
-def interpolate_environment_variables(config):
+def interpolate_environment_variables(service_dicts):
     mapping = BlankDefaultDict(os.environ)
 
     return dict(
         (service_name, process_service(service_name, service_dict, mapping))
-        for (service_name, service_dict) in config.items()
+        for (service_name, service_dict) in service_dicts.items()
     )
 
 

compose/config/service_schema.json

@@ -5,7 +5,7 @@
   "type": "object",
 
   "allOf": [
-    {"$ref": "fields_schema.json#/definitions/service"},
+    {"$ref": "fields_schema_v1.json#/definitions/service"},
     {"$ref": "#/definitions/constraints"}
   ],
 

compose/config/validation.py

@@ -74,18 +74,18 @@ def format_boolean_in_environment(instance):
     return True
 
 
-def validate_top_level_service_objects(config_file):
+def validate_top_level_service_objects(filename, service_dicts):
     """Perform some high level validation of the service name and value.
 
     This validation must happen before interpolation, which must happen
     before the rest of validation, which is why it's separate from the
     rest of the service validation.
     """
-    for service_name, service_dict in config_file.config.items():
+    for service_name, service_dict in service_dicts.items():
         if not isinstance(service_name, six.string_types):
             raise ConfigurationError(
                 "In file '{}' service name: {} needs to be a string, eg '{}'".format(
-                    config_file.filename,
+                    filename,
                     service_name,
                     service_name))
 
@@ -94,8 +94,9 @@ def validate_top_level_service_objects(config_file):
                 "In file '{}' service '{}' doesn\'t have any configuration options. "
                 "All top level keys in your docker-compose.yml must map "
                 "to a dictionary of configuration options.".format(
-                    config_file.filename,
-                    service_name))
+                    filename, service_name
+                )
+            )
 
 
 def validate_top_level_object(config_file):
@@ -105,7 +106,6 @@ def validate_top_level_object(config_file):
             "that you have defined a service at the top level.".format(
                 config_file.filename,
                 type(config_file.config)))
-    validate_top_level_service_objects(config_file)
 
 
 def validate_extends_file_path(service_name, extends_options, filename):
@@ -134,10 +134,14 @@ def anglicize_validator(validator):
     return 'a ' + validator
 
 
+def is_service_dict_schema(schema_id):
+    return schema_id == 'fields_schema_v1.json' or schema_id == '#/properties/services'
+
+
 def handle_error_for_schema_with_id(error, service_name):
     schema_id = error.schema['id']
 
-    if schema_id == 'fields_schema.json' and error.validator == 'additionalProperties':
+    if is_service_dict_schema(schema_id) and error.validator == 'additionalProperties':
         return "Invalid service name '{}' - only {} characters are allowed".format(
             # The service_name is the key to the json object
             list(error.instance)[0],
@@ -281,10 +285,11 @@ def format_error_message(error, service_name):
     return '\n'.join(format_error_message(error, service_name) for error in errors)
 
 
-def validate_against_fields_schema(config, filename):
+def validate_against_fields_schema(config, filename, version):
+    schema_filename = "fields_schema_v{0}.json".format(version)
     _validate_against_schema(
         config,
-        "fields_schema.json",
+        schema_filename,
         format_checker=["ports", "expose", "bool-value-in-mapping"],
         filename=filename)
 

compose/const.py

@@ -10,3 +10,4 @@
 LABEL_SERVICE = 'com.docker.compose.service'
 LABEL_VERSION = 'com.docker.compose.version'
 LABEL_CONFIG_HASH = 'com.docker.compose.config-hash'
+COMPOSEFILE_VERSIONS = (1, 2)