diff --git a/compose/cli/command.py b/compose/cli/command.py index 21d6ff0ddd2..b278af3a6aa 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -80,12 +80,13 @@ def get_project(base_dir, config_path=None, project_name=None, verbose=False, config_details = config.find(base_dir, config_path) api_version = '1.21' if use_networking else None - return Project.from_dicts( + return Project.from_config( get_project_name(config_details.working_dir, project_name), config.load(config_details), get_client(verbose=verbose, version=api_version), use_networking=use_networking, - network_driver=network_driver) + network_driver=network_driver + ) def get_project_name(working_dir, project_name=None): diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 734f4237b04..177d5d6c0b7 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -8,8 +8,7 @@ log = logging.getLogger(__name__) - -DEFAULT_API_VERSION = '1.19' +DEFAULT_API_VERSION = '1.21' def docker_client(version=None): diff --git a/compose/cli/main.py b/compose/cli/main.py index 4a766133fb4..006d33ecb64 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -211,11 +211,11 @@ def config(self, config_options, options): return if options['--services']: - print('\n'.join(service['name'] for service in compose_config)) + print('\n'.join(service['name'] for service in compose_config.services)) return compose_config = dict( - (service.pop('name'), service) for service in compose_config) + (service.pop('name'), service) for service in compose_config.services) print(yaml.dump( compose_config, default_flow_style=False, diff --git a/compose/config/config.py b/compose/config/config.py index 195665b5185..c77e6100854 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -10,6 +10,7 @@ import six import yaml +from ..const import COMPOSEFILE_VERSIONS from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError @@ -24,6 +25,7 @@ from .validation import validate_against_service_schema from .validation import validate_extends_file_path from .validation import validate_top_level_object +from .validation import validate_top_level_service_objects DOCKER_CONFIG_KEYS = [ @@ -116,6 +118,20 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): def from_filename(cls, filename): return cls(filename, load_yaml(filename)) + def get_service_dicts(self, version): + return self.config if version == 1 else self.config.get('services', {}) + + +class Config(namedtuple('_Config', 'version services volumes')): + """ + :param version: configuration version + :type version: int + :param services: List of service description dictionaries + :type services: :class:`list` + :param volumes: List of volume description dictionaries + :type volumes: :class:`list` + """ + class ServiceConfig(namedtuple('_ServiceConfig', 'working_dir filename name config')): @@ -148,6 +164,34 @@ def find(base_dir, filenames): [ConfigFile.from_filename(f) for f in filenames]) +def get_config_version(config_details): + def get_version(config): + if config.config is None: + return 1 + version = config.config.get('version', 1) + if isinstance(version, dict): + # in that case 'version' is probably a service name, so assume + # this is a legacy (version=1) file + version = 1 + return version + + main_file = config_details.config_files[0] + validate_top_level_object(main_file) + version = get_version(main_file) + for next_file in config_details.config_files[1:]: + validate_top_level_object(next_file) + next_file_version = get_version(next_file) + + if version != next_file_version and next_file_version is not None: + raise ConfigurationError( + "Version mismatch: main file {0} specifies version {1} but " + "extension file {2} uses version {3}".format( + main_file.filename, version, next_file.filename, next_file_version + ) + ) + return version + + def get_default_config_files(base_dir): (candidates, path) = find_candidates_in_parent_dirs(SUPPORTED_FILENAMES, base_dir) @@ -194,14 +238,52 @@ def load(config_details): Return a fully interpolated, extended and validated configuration. """ + version = get_config_version(config_details) + if version not in COMPOSEFILE_VERSIONS: + raise ConfigurationError('Invalid config version provided: {0}'.format(version)) + + processed_files = [] + for config_file in config_details.config_files: + processed_files.append( + process_config_file(config_file, version=version) + ) + config_details = config_details._replace(config_files=processed_files) + + if version == 1: + service_dicts = load_services( + config_details.working_dir, config_details.config_files, + version + ) + volumes = {} + elif version == 2: + config_files = [ + ConfigFile(f.filename, f.config.get('services', {})) + for f in config_details.config_files + ] + service_dicts = load_services( + config_details.working_dir, config_files, version + ) + volumes = load_volumes(config_details.config_files) + + return Config(version, service_dicts, volumes) + + +def load_volumes(config_files): + volumes = {} + for config_file in config_files: + for name, volume_config in config_file.config.get('volumes', {}).items(): + volumes.update({name: volume_config}) + return volumes + +def load_services(working_dir, config_files, version): def build_service(filename, service_name, service_dict): service_config = ServiceConfig.with_abs_paths( - config_details.working_dir, + working_dir, filename, service_name, service_dict) - resolver = ServiceExtendsResolver(service_config) + resolver = ServiceExtendsResolver(service_config, version) service_dict = process_service(resolver.run()) # TODO: move to validate_service() @@ -227,20 +309,28 @@ def merge_services(base, override): for name in all_service_names } - config_file = process_config_file(config_details.config_files[0]) - for next_file in config_details.config_files[1:]: - next_file = process_config_file(next_file) - + config_file = config_files[0] + for next_file in config_files[1:]: config = merge_services(config_file.config, next_file.config) config_file = config_file._replace(config=config) return build_services(config_file) -def process_config_file(config_file, service_name=None): - validate_top_level_object(config_file) - processed_config = interpolate_environment_variables(config_file.config) - validate_against_fields_schema(processed_config, config_file.filename) +def process_config_file(config_file, version, service_name=None): + service_dicts = config_file.get_service_dicts(version) + validate_top_level_service_objects( + config_file.filename, service_dicts + ) + interpolated_config = interpolate_environment_variables(service_dicts) + if version == 2: + processed_config = dict(config_file.config) + processed_config.update({'services': interpolated_config}) + if version == 1: + processed_config = interpolated_config + validate_against_fields_schema( + processed_config, config_file.filename, version + ) if service_name and service_name not in processed_config: raise ConfigurationError( @@ -251,10 +341,11 @@ def process_config_file(config_file, service_name=None): class ServiceExtendsResolver(object): - def __init__(self, service_config, already_seen=None): + def __init__(self, service_config, version, already_seen=None): self.service_config = service_config self.working_dir = service_config.working_dir self.already_seen = already_seen or [] + self.version = version @property def signature(self): @@ -283,7 +374,8 @@ def validate_and_construct_extends(self): extended_file = process_config_file( ConfigFile.from_filename(config_path), - service_name=service_name) + version=self.version, service_name=service_name + ) service_config = extended_file.config[service_name] return config_path, service_config, service_name @@ -294,6 +386,7 @@ def resolve_extends(self, extended_config_path, service_dict, service_name): extended_config_path, service_name, service_dict), + self.version, already_seen=self.already_seen + [self.signature]) service_config = resolver.run() diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema_v1.json similarity index 99% rename from compose/config/fields_schema.json rename to compose/config/fields_schema_v1.json index fdf56fd91d1..6f0a3631988 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema_v1.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", - "id": "fields_schema.json", + "id": "fields_schema_v1.json", "patternProperties": { "^[a-zA-Z0-9._-]+$": { diff --git a/compose/config/fields_schema_v2.json b/compose/config/fields_schema_v2.json new file mode 100644 index 00000000000..49cab367084 --- /dev/null +++ b/compose/config/fields_schema_v2.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "id": "fields_schema_v2.json", + + "properties": { + "version": { + "enum": [2] + }, + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "fields_schema_v1.json#/definitions/service" + } + }, + "additionalProperties": false + }, + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + } + }, + + "definitions": { + "volume": { + "id": "#/definitions/volume", + "type": "object", + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + }, + "additionalProperties": false + } + } + } + }, + "additionalProperties": false +} diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index ba7e35c1e58..12eb497b603 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -8,12 +8,12 @@ log = logging.getLogger(__name__) -def interpolate_environment_variables(config): +def interpolate_environment_variables(service_dicts): mapping = BlankDefaultDict(os.environ) return dict( (service_name, process_service(service_name, service_dict, mapping)) - for (service_name, service_dict) in config.items() + for (service_name, service_dict) in service_dicts.items() ) diff --git a/compose/config/service_schema.json b/compose/config/service_schema.json index 05774efdda7..91a1e005053 100644 --- a/compose/config/service_schema.json +++ b/compose/config/service_schema.json @@ -5,7 +5,7 @@ "type": "object", "allOf": [ - {"$ref": "fields_schema.json#/definitions/service"}, + {"$ref": "fields_schema_v1.json#/definitions/service"}, {"$ref": "#/definitions/constraints"} ], diff --git a/compose/config/validation.py b/compose/config/validation.py index d16bdb9d3e0..091014f6567 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -74,18 +74,18 @@ def format_boolean_in_environment(instance): return True -def validate_top_level_service_objects(config_file): +def validate_top_level_service_objects(filename, service_dicts): """Perform some high level validation of the service name and value. This validation must happen before interpolation, which must happen before the rest of validation, which is why it's separate from the rest of the service validation. """ - for service_name, service_dict in config_file.config.items(): + for service_name, service_dict in service_dicts.items(): if not isinstance(service_name, six.string_types): raise ConfigurationError( "In file '{}' service name: {} needs to be a string, eg '{}'".format( - config_file.filename, + filename, service_name, service_name)) @@ -94,8 +94,9 @@ def validate_top_level_service_objects(config_file): "In file '{}' service '{}' doesn\'t have any configuration options. " "All top level keys in your docker-compose.yml must map " "to a dictionary of configuration options.".format( - config_file.filename, - service_name)) + filename, service_name + ) + ) def validate_top_level_object(config_file): @@ -105,7 +106,6 @@ def validate_top_level_object(config_file): "that you have defined a service at the top level.".format( config_file.filename, type(config_file.config))) - validate_top_level_service_objects(config_file) def validate_extends_file_path(service_name, extends_options, filename): @@ -134,10 +134,14 @@ def anglicize_validator(validator): return 'a ' + validator +def is_service_dict_schema(schema_id): + return schema_id == 'fields_schema_v1.json' or schema_id == '#/properties/services' + + def handle_error_for_schema_with_id(error, service_name): schema_id = error.schema['id'] - if schema_id == 'fields_schema.json' and error.validator == 'additionalProperties': + if is_service_dict_schema(schema_id) and error.validator == 'additionalProperties': return "Invalid service name '{}' - only {} characters are allowed".format( # The service_name is the key to the json object list(error.instance)[0], @@ -281,10 +285,11 @@ def format_error_message(error, service_name): return '\n'.join(format_error_message(error, service_name) for error in errors) -def validate_against_fields_schema(config, filename): +def validate_against_fields_schema(config, filename, version): + schema_filename = "fields_schema_v{0}.json".format(version) _validate_against_schema( config, - "fields_schema.json", + schema_filename, format_checker=["ports", "expose", "bool-value-in-mapping"], filename=filename) diff --git a/compose/const.py b/compose/const.py index 1b6894189e2..9c607ca26d8 100644 --- a/compose/const.py +++ b/compose/const.py @@ -10,3 +10,4 @@ LABEL_SERVICE = 'com.docker.compose.service' LABEL_VERSION = 'com.docker.compose.version' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' +COMPOSEFILE_VERSIONS = (1, 2) diff --git a/compose/container.py b/compose/container.py index 8f96a9447d2..5730f224618 100644 --- a/compose/container.py +++ b/compose/container.py @@ -177,6 +177,12 @@ def get_local_port(self, port, protocol='tcp'): port = self.ports.get("%s/%s" % (port, protocol)) return "{HostIp}:{HostPort}".format(**port[0]) if port else None + def get_mount(self, mount_dest): + for mount in self.get('Mounts'): + if mount['Destination'] == mount_dest: + return mount + return None + def start(self, **options): return self.client.start(self.id, **options) @@ -222,16 +228,6 @@ def inspect(self): self.has_been_inspected = True return self.dictionary - # TODO: only used by tests, move to test module - def links(self): - links = [] - for container in self.client.containers(): - for name in container['Names']: - bits = name.split('/') - if len(bits) > 2 and bits[1] == self.name: - links.append(bits[2]) - return links - def attach(self, *args, **kwargs): return self.client.attach(self.id, *args, **kwargs) diff --git a/compose/project.py b/compose/project.py index 76dccfe20f7..3801bbb9f87 100644 --- a/compose/project.py +++ b/compose/project.py @@ -20,6 +20,7 @@ from .service import Net from .service import Service from .service import ServiceNet +from .volume import Volume log = logging.getLogger(__name__) @@ -29,12 +30,13 @@ class Project(object): """ A collection of services. """ - def __init__(self, name, services, client, use_networking=False, network_driver=None): + def __init__(self, name, services, client, volumes=None, use_networking=False, network_driver=None): self.name = name self.services = services self.client = client self.use_networking = use_networking self.network_driver = network_driver + self.volumes = volumes or [] def labels(self, one_off=False): return [ @@ -43,16 +45,16 @@ def labels(self, one_off=False): ] @classmethod - def from_dicts(cls, name, service_dicts, client, use_networking=False, network_driver=None): + def from_config(cls, name, config_data, client, use_networking=False, network_driver=None): """ - Construct a ServiceCollection from a list of dicts representing services. + Construct a Project from a config.Config object. """ project = cls(name, [], client, use_networking=use_networking, network_driver=network_driver) if use_networking: - remove_links(service_dicts) + remove_links(config_data.services) - for service_dict in service_dicts: + for service_dict in config_data.services: links = project.get_links(service_dict) volumes_from = project.get_volumes_from(service_dict) net = project.get_net(service_dict) @@ -66,6 +68,14 @@ def from_dicts(cls, name, service_dicts, client, use_networking=False, network_d net=net, volumes_from=volumes_from, **service_dict)) + if config_data.volumes: + for vol_name, data in config_data.volumes.items(): + project.volumes.append( + Volume( + client=client, project=name, name=vol_name, + driver=data.get('driver'), driver_opts=data.get('driver_opts') + ) + ) return project @property @@ -218,6 +228,27 @@ def kill(self, service_names=None, **options): def remove_stopped(self, service_names=None, **options): parallel.parallel_remove(self.containers(service_names, stopped=True), options) + def initialize_volumes(self): + try: + for volume in self.volumes: + volume.create() + except NotFound: + raise ConfigurationError( + 'Volume %s specifies nonexistent driver %s' % (volume.name, volume.driver) + ) + except APIError as e: + if 'Choose a different volume name' in str(e): + raise ConfigurationError( + 'Configuration for volume {0} specifies driver {1}, but ' + 'a volume with the same name uses a different driver ' + '({3}). If you wish to use the new configuration, please ' + 'remove the existing volume "{2}" first:\n' + '$ docker volume rm {2}'.format( + volume.name, volume.driver, volume.full_name, + volume.inspect()['Driver'] + ) + ) + def restart(self, service_names=None, **options): containers = self.containers(service_names, stopped=True) parallel.parallel_restart(containers, options) @@ -253,6 +284,8 @@ def up(self, if self.use_networking and self.uses_default_network(): self.ensure_network_exists() + self.initialize_volumes() + return [ container for service in services diff --git a/compose/service.py b/compose/service.py index d1509871004..24fa63942dc 100644 --- a/compose/service.py +++ b/compose/service.py @@ -849,7 +849,13 @@ def get_container_data_volumes(container, volumes_option): a mapping of volume bindings for those volumes. """ volumes = [] - container_volumes = container.get('Volumes') or {} + volumes_option = volumes_option or [] + + container_mounts = dict( + (mount['Destination'], mount) + for mount in container.get('Mounts') or {} + ) + image_volumes = [ VolumeSpec.parse(volume) for volume in @@ -861,13 +867,14 @@ def get_container_data_volumes(container, volumes_option): if volume.external: continue - volume_path = container_volumes.get(volume.internal) + mount = container_mounts.get(volume.internal) + # New volume, doesn't exist in the old container - if not volume_path: + if not mount: continue # Copy existing volume from old container - volume = volume._replace(external=volume_path) + volume = volume._replace(external=mount['Source']) volumes.append(volume) return volumes diff --git a/compose/volume.py b/compose/volume.py new file mode 100644 index 00000000000..055bd6abde4 --- /dev/null +++ b/compose/volume.py @@ -0,0 +1,25 @@ +from __future__ import unicode_literals + + +class Volume(object): + def __init__(self, client, project, name, driver=None, driver_opts=None): + self.client = client + self.project = project + self.name = name + self.driver = driver + self.driver_opts = driver_opts + + def create(self): + return self.client.create_volume( + self.full_name, self.driver, self.driver_opts + ) + + def remove(self): + return self.client.remove_volume(self.full_name) + + def inspect(self): + return self.client.inspect_volume(self.full_name) + + @property + def full_name(self): + return '{0}_{1}'.format(self.project, self.name) diff --git a/docker-compose.spec b/docker-compose.spec index 24d03e05b2e..c760d7b4ce5 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -18,8 +18,13 @@ exe = EXE(pyz, a.datas, [ ( - 'compose/config/fields_schema.json', - 'compose/config/fields_schema.json', + 'compose/config/fields_schema_v1.json', + 'compose/config/fields_schema_v1.json', + 'DATA' + ), + ( + 'compose/config/fields_schema_v2.json', + 'compose/config/fields_schema_v2.json', 'DATA' ), ( @@ -33,6 +38,7 @@ exe = EXE(pyz, 'DATA' ) ], + name='docker-compose', debug=False, strip=None, diff --git a/docs/compose-file.md b/docs/compose-file.md index 2a6028b8cf2..29e0c647c3c 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -24,6 +24,64 @@ As with `docker run`, options specified in the Dockerfile (e.g., `CMD`, `EXPOSE`, `VOLUME`, `ENV`) are respected by default - you don't need to specify them again in `docker-compose.yml`. +## Versioning + +It is possible to use different versions of the `compose.yml` format. +Below are the formats currently supported by compose. + + +### Version 1 + +Compose files that do not declare a version are considered "version 1". In +those files, all the [services](#service-configuration-reference) are declared +at the root of the document. + +Version 1 files do not support the declaration of +named [volumes](#volume-configuration-reference) + +Example: + + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + - logvolume01:/var/log + links: + - redis + redis: + image: redis + + +### Version 2 + +Compose files using the version 2 syntax must indicate the version number at +the root of the document. All [services](#service-configuration-reference) +must be declared under the `services` key. +Named [volumes](#volume-configuration-reference) must be declared under the +`volumes` key. + +Example: + + version: 2 + services: + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + - logvolume01:/var/log + links: + - redis + redis: + image: redis + volumes: + logvolume01: + driver: default + + ## Service configuration reference This section contains a list of all configuration options supported by a service @@ -413,6 +471,34 @@ Each of these is a single value, analogous to its stdin_open: true tty: true + +## Volume configuration reference + +While it is possible to declare volumes on the fly as part of the service +declaration, this section allows you to create named volumes that can be +reused across multiple services (without relying on `volumes_from`), and are +easily retrieved and inspected using the docker command line or API. +See the [docker volume](http://docs.docker.com/reference/commandline/volume/) +subcommand documentation for more information. + +### driver + +Specify which volume driver should be used for this volume. Defaults to +`local`. An exception will be raised if the driver is not available. + + driver: foobar + +### driver_opts + +Specify a list of options as key-value pairs to pass to the driver for this +volume. Those options are driver dependent - consult the driver's +documentation for more information. Optional. + + driver_opts: + foo: "bar" + baz: 1 + + ## Variable substitution Your configuration options can contain environment variables. Compose uses the diff --git a/docs/index.md b/docs/index.md index 36b93a39ead..6e8f2090c6a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -31,16 +31,22 @@ they can be run together in an isolated environment. A `docker-compose.yml` looks like this: - web: - build: . - ports: - - "5000:5000" - volumes: - - .:/code - links: - - redis - redis: - image: redis + version: 2 + services: + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + - logvolume01:/var/log + links: + - redis + redis: + image: redis + volumes: + logvolume01: + driver: default For more information about the Compose file, see the [Compose file reference](compose-file.md) diff --git a/requirements.txt b/requirements.txt index 659cb57f4ed..8c6d5f3ad40 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML==3.11 -docker-py==1.5.0 +docker-py==1.6.0 dockerpty==0.3.4 docopt==0.6.1 enum34==1.0.4 diff --git a/script/test-versions b/script/test-versions index 623b107b930..76e55e1193b 100755 --- a/script/test-versions +++ b/script/test-versions @@ -18,7 +18,8 @@ get_versions="docker run --rm if [ "$DOCKER_VERSIONS" == "" ]; then DOCKER_VERSIONS="$($get_versions default)" elif [ "$DOCKER_VERSIONS" == "all" ]; then - DOCKER_VERSIONS="$($get_versions recent -n 2)" + # TODO: `-n 2` when engine 1.10 releases + DOCKER_VERSIONS="$($get_versions recent -n 1)" fi diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index e6fa38a8b5e..1885727a13b 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -16,6 +16,7 @@ from compose.cli.docker_client import docker_client from compose.container import Container from tests.integration.testcases import DockerClientTestCase +from tests.integration.testcases import get_links from tests.integration.testcases import pull_busybox @@ -871,7 +872,7 @@ def test_home_and_env_var_in_volume_path(self): self.dispatch(['up', '-d'], None) container = self.project.containers(stopped=True)[0] - actual_host_path = container.get('Volumes')['/container-path'] + actual_host_path = container.get_mount('/container-path')['Source'] components = actual_host_path.split('/') assert components[-2:] == ['home-dir', 'my-volume'] @@ -909,7 +910,7 @@ def test_up_with_multiple_files(self): web, other, db = containers self.assertEqual(web.human_readable_command, 'top') - self.assertTrue({'db', 'other'} <= set(web.links())) + self.assertTrue({'db', 'other'} <= set(get_links(web))) self.assertEqual(db.human_readable_command, 'top') self.assertEqual(other.human_readable_command, 'top') @@ -931,7 +932,9 @@ def test_up_with_extends(self): self.assertEqual(len(containers), 2) web = containers[1] - self.assertEqual(set(web.links()), set(['db', 'mydb_1', 'extends_mydb_1'])) + self.assertEqual( + set(get_links(web)), + set(['db', 'mydb_1', 'extends_mydb_1'])) expected_env = set([ "FOO=1", diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 229f653a437..d51830bb0ca 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +import random + from .testcases import DockerClientTestCase from compose.cli.docker_client import docker_client from compose.config import config @@ -69,9 +71,9 @@ def test_volumes_from_service(self): 'volumes_from': ['data'], }, }) - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=service_dicts, + config_data=service_dicts, client=self.client, ) db = project.get_service('db') @@ -86,9 +88,9 @@ def test_volumes_from_container(self): name='composetest_data_container', labels={LABEL_PROJECT: 'composetest'}, ) - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=build_service_dicts({ + config_data=build_service_dicts({ 'db': { 'image': 'busybox:latest', 'volumes_from': ['composetest_data_container'], @@ -117,9 +119,9 @@ def test_get_network(self): assert project.get_network()['Name'] == network_name def test_net_from_service(self): - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=build_service_dicts({ + config_data=build_service_dicts({ 'net': { 'image': 'busybox:latest', 'command': ["top"] @@ -149,9 +151,9 @@ def test_net_from_container(self): ) net_container.start() - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=build_service_dicts({ + config_data=build_service_dicts({ 'web': { 'image': 'busybox:latest', 'net': 'container:composetest_net_container' @@ -331,15 +333,17 @@ def test_project_up_with_no_recreate_running(self): project.up(['db']) self.assertEqual(len(project.containers()), 1) old_db_id = project.containers()[0].id - db_volume_path = project.containers()[0].inspect()['Volumes']['/var/db'] + container, = project.containers() + db_volume_path = container.get_mount('/var/db')['Source'] project.up(strategy=ConvergenceStrategy.never) self.assertEqual(len(project.containers()), 2) db_container = [c for c in project.containers() if 'db' in c.name][0] self.assertEqual(db_container.id, old_db_id) - self.assertEqual(db_container.inspect()['Volumes']['/var/db'], - db_volume_path) + self.assertEqual( + db_container.get_mount('/var/db')['Source'], + db_volume_path) def test_project_up_with_no_recreate_stopped(self): web = self.create_service('web') @@ -354,8 +358,9 @@ def test_project_up_with_no_recreate_stopped(self): old_containers = project.containers(stopped=True) self.assertEqual(len(old_containers), 1) - old_db_id = old_containers[0].id - db_volume_path = old_containers[0].inspect()['Volumes']['/var/db'] + old_container, = old_containers + old_db_id = old_container.id + db_volume_path = old_container.get_mount('/var/db')['Source'] project.up(strategy=ConvergenceStrategy.never) @@ -365,8 +370,9 @@ def test_project_up_with_no_recreate_stopped(self): db_container = [c for c in new_containers if 'db' in c.name][0] self.assertEqual(db_container.id, old_db_id) - self.assertEqual(db_container.inspect()['Volumes']['/var/db'], - db_volume_path) + self.assertEqual( + db_container.get_mount('/var/db')['Source'], + db_volume_path) def test_project_up_without_all_services(self): console = self.create_service('console') @@ -396,9 +402,9 @@ def test_project_up_starts_links(self): self.assertEqual(len(console.containers()), 0) def test_project_up_starts_depends(self): - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=build_service_dicts({ + config_data=build_service_dicts({ 'console': { 'image': 'busybox:latest', 'command': ["top"], @@ -431,9 +437,9 @@ def test_project_up_starts_depends(self): self.assertEqual(len(project.get_service('console').containers()), 0) def test_project_up_with_no_deps(self): - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=build_service_dicts({ + config_data=build_service_dicts({ 'console': { 'image': 'busybox:latest', 'command': ["top"], @@ -504,3 +510,119 @@ def test_unscale_after_restart(self): project.up() service = project.get_service('web') self.assertEqual(len(service.containers()), 1) + + def test_project_up_volumes(self): + vol_name = '{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) + config_data = config.Config( + version=2, services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], volumes={vol_name: {'driver': 'local'}} + ) + + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + project.up() + self.assertEqual(len(project.containers()), 1) + + volume_data = self.client.inspect_volume(full_vol_name) + self.assertEqual(volume_data['Name'], full_vol_name) + self.assertEqual(volume_data['Driver'], 'local') + + def test_initialize_volumes(self): + vol_name = '{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) + config_data = config.Config( + version=2, services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], volumes={vol_name: {}} + ) + + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + project.initialize_volumes() + + volume_data = self.client.inspect_volume(full_vol_name) + self.assertEqual(volume_data['Name'], full_vol_name) + self.assertEqual(volume_data['Driver'], 'local') + + def test_project_up_implicit_volume_driver(self): + vol_name = '{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) + config_data = config.Config( + version=2, services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], volumes={vol_name: {}} + ) + + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + project.up() + + volume_data = self.client.inspect_volume(full_vol_name) + self.assertEqual(volume_data['Name'], full_vol_name) + self.assertEqual(volume_data['Driver'], 'local') + + def test_project_up_invalid_volume_driver(self): + vol_name = '{0:x}'.format(random.getrandbits(32)) + + config_data = config.Config( + version=2, services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], volumes={vol_name: {'driver': 'foobar'}} + ) + + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + with self.assertRaises(config.ConfigurationError): + project.initialize_volumes() + + def test_project_up_updated_driver(self): + vol_name = '{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) + + config_data = config.Config( + version=2, services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], volumes={vol_name: {'driver': 'local'}} + ) + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + project.initialize_volumes() + + volume_data = self.client.inspect_volume(full_vol_name) + self.assertEqual(volume_data['Name'], full_vol_name) + self.assertEqual(volume_data['Driver'], 'local') + + config_data = config_data._replace( + volumes={vol_name: {'driver': 'smb'}} + ) + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + with self.assertRaises(config.ConfigurationError) as e: + project.initialize_volumes() + assert 'Configuration for volume {0} specifies driver smb'.format( + vol_name + ) in str(e.exception) diff --git a/tests/integration/resilience_test.py b/tests/integration/resilience_test.py index 7f75356d829..5df751c770d 100644 --- a/tests/integration/resilience_test.py +++ b/tests/integration/resilience_test.py @@ -18,12 +18,12 @@ def setUp(self): container = self.db.create_container() container.start() - self.host_path = container.get('Volumes')['/var/db'] + self.host_path = container.get_mount('/var/db')['Source'] def test_successful_recreate(self): self.project.up(strategy=ConvergenceStrategy.always) container = self.db.containers()[0] - self.assertEqual(container.get('Volumes')['/var/db'], self.host_path) + self.assertEqual(container.get_mount('/var/db')['Source'], self.host_path) def test_create_failure(self): with mock.patch('compose.service.Service.create_container', crash): @@ -32,7 +32,7 @@ def test_create_failure(self): self.project.up() container = self.db.containers()[0] - self.assertEqual(container.get('Volumes')['/var/db'], self.host_path) + self.assertEqual(container.get_mount('/var/db')['Source'], self.host_path) def test_start_failure(self): with mock.patch('compose.container.Container.start', crash): @@ -41,7 +41,7 @@ def test_start_failure(self): self.project.up() container = self.db.containers()[0] - self.assertEqual(container.get('Volumes')['/var/db'], self.host_path) + self.assertEqual(container.get_mount('/var/db')['Source'], self.host_path) class Crash(Exception): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index ff5716568d1..4a0eaacb4fb 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -12,6 +12,7 @@ from .. import mock from .testcases import DockerClientTestCase +from .testcases import get_links from .testcases import pull_busybox from compose import __version__ from compose.config.types import VolumeFromSpec @@ -88,13 +89,13 @@ def test_create_container_with_unspecified_volume(self): service = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')]) container = service.create_container() container.start() - self.assertIn('/var/db', container.get('Volumes')) + assert container.get_mount('/var/db') def test_create_container_with_volume_driver(self): service = self.create_service('db', volume_driver='foodriver') container = service.create_container() container.start() - self.assertEqual('foodriver', container.get('Config.VolumeDriver')) + self.assertEqual('foodriver', container.get('HostConfig.VolumeDriver')) def test_create_container_with_cpu_shares(self): service = self.create_service('db', cpu_shares=73) @@ -158,12 +159,11 @@ def test_create_container_with_specified_volume(self): volumes=[VolumeSpec(host_path, container_path, 'rw')]) container = service.create_container() container.start() - - volumes = container.inspect()['Volumes'] - self.assertIn(container_path, volumes) + assert container.get_mount(container_path) # Match the last component ("host-path"), because boot2docker symlinks /tmp - actual_host_path = volumes[container_path] + actual_host_path = container.get_mount(container_path)['Source'] + self.assertTrue(path.basename(actual_host_path) == path.basename(host_path), msg=("Last component differs: %s, %s" % (actual_host_path, host_path))) @@ -173,10 +173,10 @@ def test_recreate_preserves_volume_with_trailing_slash(self): """ service = self.create_service('data', volumes=[VolumeSpec.parse('/data/')]) old_container = create_and_start_container(service) - volume_path = old_container.get('Volumes')['/data'] + volume_path = old_container.get_mount('/data')['Source'] new_container = service.recreate_container(old_container) - self.assertEqual(new_container.get('Volumes')['/data'], volume_path) + self.assertEqual(new_container.get_mount('/data')['Source'], volume_path) def test_duplicate_volume_trailing_slash(self): """ @@ -250,7 +250,7 @@ def test_execute_convergence_plan_recreate(self): self.assertEqual(old_container.name, 'composetest_db_1') old_container.start() old_container.inspect() # reload volume data - volume_path = old_container.get('Volumes')['/etc'] + volume_path = old_container.get_mount('/etc')['Source'] num_containers_before = len(self.client.containers(all=True)) @@ -262,7 +262,7 @@ def test_execute_convergence_plan_recreate(self): self.assertEqual(new_container.get('Config.Cmd'), ['-d', '1']) self.assertIn('FOO=2', new_container.get('Config.Env')) self.assertEqual(new_container.name, 'composetest_db_1') - self.assertEqual(new_container.get('Volumes')['/etc'], volume_path) + self.assertEqual(new_container.get_mount('/etc')['Source'], volume_path) self.assertIn( 'affinity:container==%s' % old_container.id, new_container.get('Config.Env')) @@ -305,14 +305,19 @@ def test_execute_convergence_plan_with_image_declared_volume(self): ) old_container = create_and_start_container(service) - self.assertEqual(list(old_container.get('Volumes').keys()), ['/data']) - volume_path = old_container.get('Volumes')['/data'] + self.assertEqual( + [mount['Destination'] for mount in old_container.get('Mounts')], ['/data'] + ) + volume_path = old_container.get_mount('/data')['Source'] new_container, = service.execute_convergence_plan( ConvergencePlan('recreate', [old_container])) - self.assertEqual(list(new_container.get('Volumes')), ['/data']) - self.assertEqual(new_container.get('Volumes')['/data'], volume_path) + self.assertEqual( + [mount['Destination'] for mount in new_container.get('Mounts')], + ['/data'] + ) + self.assertEqual(new_container.get_mount('/data')['Source'], volume_path) def test_execute_convergence_plan_when_image_volume_masks_config(self): service = self.create_service( @@ -321,8 +326,11 @@ def test_execute_convergence_plan_when_image_volume_masks_config(self): ) old_container = create_and_start_container(service) - self.assertEqual(list(old_container.get('Volumes').keys()), ['/data']) - volume_path = old_container.get('Volumes')['/data'] + self.assertEqual( + [mount['Destination'] for mount in old_container.get('Mounts')], + ['/data'] + ) + volume_path = old_container.get_mount('/data')['Source'] service.options['volumes'] = [VolumeSpec.parse('/tmp:/data')] @@ -336,8 +344,11 @@ def test_execute_convergence_plan_when_image_volume_masks_config(self): "Service \"db\" is using volume \"/data\" from the previous container", args[0]) - self.assertEqual(list(new_container.get('Volumes')), ['/data']) - self.assertEqual(new_container.get('Volumes')['/data'], volume_path) + self.assertEqual( + [mount['Destination'] for mount in new_container.get('Mounts')], + ['/data'] + ) + self.assertEqual(new_container.get_mount('/data')['Source'], volume_path) def test_execute_convergence_plan_without_start(self): service = self.create_service( @@ -376,7 +387,7 @@ def test_start_container_creates_links(self): create_and_start_container(web) self.assertEqual( - set(web.containers()[0].links()), + set(get_links(web.containers()[0])), set([ 'composetest_db_1', 'db_1', 'composetest_db_2', 'db_2', @@ -392,7 +403,7 @@ def test_start_container_creates_links_with_names(self): create_and_start_container(web) self.assertEqual( - set(web.containers()[0].links()), + set(get_links(web.containers()[0])), set([ 'composetest_db_1', 'db_1', 'composetest_db_2', 'db_2', @@ -410,7 +421,7 @@ def test_start_container_with_external_links(self): create_and_start_container(web) self.assertEqual( - set(web.containers()[0].links()), + set(get_links(web.containers()[0])), set([ 'composetest_db_1', 'composetest_db_2', @@ -424,7 +435,7 @@ def test_start_normal_container_does_not_create_links_to_its_own_service(self): create_and_start_container(db) c = create_and_start_container(db) - self.assertEqual(set(c.links()), set([])) + self.assertEqual(set(get_links(c)), set([])) def test_start_one_off_container_creates_links_to_its_own_service(self): db = self.create_service('db') @@ -435,7 +446,7 @@ def test_start_one_off_container_creates_links_to_its_own_service(self): c = create_and_start_container(db, one_off=True) self.assertEqual( - set(c.links()), + set(get_links(c)), set([ 'composetest_db_1', 'db_1', 'composetest_db_2', 'db_2', diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 1fecce87b84..d07dfa82a02 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -7,6 +7,7 @@ import py from .testcases import DockerClientTestCase +from .testcases import get_links from compose.config import config from compose.project import Project from compose.service import ConvergenceStrategy @@ -25,10 +26,10 @@ def make_project(self, cfg): details = config.ConfigDetails( 'working_dir', [config.ConfigFile(None, cfg)]) - return Project.from_dicts( + return Project.from_config( name='composetest', client=self.client, - service_dicts=config.load(details)) + config_data=config.load(details)) class BasicProjectTest(ProjectTestCase): @@ -186,8 +187,8 @@ def test_service_recreated_when_dependency_created(self): web, = [c for c in containers if c.service == 'web'] nginx, = [c for c in containers if c.service == 'nginx'] - self.assertEqual(web.links(), ['composetest_db_1', 'db', 'db_1']) - self.assertEqual(nginx.links(), ['composetest_web_1', 'web', 'web_1']) + self.assertEqual(set(get_links(web)), {'composetest_db_1', 'db', 'db_1'}) + self.assertEqual(set(get_links(nginx)), {'composetest_web_1', 'web', 'web_1'}) class ServiceStateTest(DockerClientTestCase): diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 9ea68e39c53..8e0525eefd6 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -16,6 +16,16 @@ def pull_busybox(client): client.pull('busybox:latest', stream=False) +def get_links(container): + links = container.get('HostConfig.Links') or [] + + def format_link(link): + _, alias = link.split(':') + return alias.split('/')[-1] + + return [format_link(link) for link in links] + + class DockerClientTestCase(unittest.TestCase): @classmethod def setUpClass(cls): @@ -25,11 +35,14 @@ def tearDown(self): for c in self.client.containers( all=True, filters={'label': '%s=composetest' % LABEL_PROJECT}): - self.client.kill(c['Id']) - self.client.remove_container(c['Id']) + self.client.remove_container(c['Id'], force=True) for i in self.client.images( filters={'label': 'com.docker.compose.test_image'}): self.client.remove_image(i) + volumes = self.client.volumes().get('Volumes') or [] + for v in volumes: + if 'composetest_' in v['Name']: + self.client.remove_volume(v['Name']) def create_service(self, name, **kwargs): if 'image' not in kwargs and 'build' not in kwargs: diff --git a/tests/integration/volume_test.py b/tests/integration/volume_test.py new file mode 100644 index 00000000000..b60860400a3 --- /dev/null +++ b/tests/integration/volume_test.py @@ -0,0 +1,55 @@ +from __future__ import unicode_literals + +from docker.errors import DockerException + +from .testcases import DockerClientTestCase +from compose.volume import Volume + + +class VolumeTest(DockerClientTestCase): + def setUp(self): + self.tmp_volumes = [] + + def tearDown(self): + for volume in self.tmp_volumes: + try: + self.client.remove_volume(volume.full_name) + except DockerException: + pass + + def create_volume(self, name, driver=None, opts=None): + vol = Volume( + self.client, 'composetest', name, driver=driver, driver_opts=opts + ) + self.tmp_volumes.append(vol) + return vol + + def test_create_volume(self): + vol = self.create_volume('volume01') + vol.create() + info = self.client.inspect_volume(vol.full_name) + assert info['Name'] == vol.full_name + + def test_recreate_existing_volume(self): + vol = self.create_volume('volume01') + + vol.create() + info = self.client.inspect_volume(vol.full_name) + assert info['Name'] == vol.full_name + + vol.create() + info = self.client.inspect_volume(vol.full_name) + assert info['Name'] == vol.full_name + + def test_inspect_volume(self): + vol = self.create_volume('volume01') + vol.create() + info = vol.inspect() + assert info['Name'] == vol.full_name + + def test_remove_volume(self): + vol = Volume(self.client, 'composetest', 'volume01') + vol.create() + vol.remove() + volumes = self.client.volumes()['Volumes'] + assert len([v for v in volumes if v['Name'] == vol.full_name]) == 0 diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e975cb9d853..281e81d1e77 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -26,7 +26,7 @@ def make_service_dict(name, service_dict, working_dir, filename=None): working_dir=working_dir, filename=filename, name=name, - config=service_dict)) + config=service_dict), version=1) return config.process_service(resolver.run()) @@ -51,8 +51,41 @@ def test_load(self): 'tests/fixtures/extends', 'common.yml' ) + ).services + + self.assertEqual( + service_sort(service_dicts), + service_sort([ + { + 'name': 'bar', + 'image': 'busybox', + 'environment': {'FOO': '1'}, + }, + { + 'name': 'foo', + 'image': 'busybox', + } + ]) ) + def test_load_v2(self): + config_data = config.load( + build_config_details({ + 'version': 2, + 'services': { + 'foo': {'image': 'busybox'}, + 'bar': {'image': 'busybox', 'environment': ['FOO=1']}, + }, + 'volumes': { + 'hello': { + 'driver': 'default', + 'driver_opts': {'beep': 'boop'} + } + } + }, 'working_dir', 'filename.yml') + ) + service_dicts = config_data.services + volume_dict = config_data.volumes self.assertEqual( service_sort(service_dicts), service_sort([ @@ -67,6 +100,52 @@ def test_load(self): } ]) ) + self.assertEqual(volume_dict, { + 'hello': { + 'driver': 'default', + 'driver_opts': {'beep': 'boop'} + } + }) + + def test_load_service_with_name_version(self): + config_data = config.load( + build_config_details({ + 'version': { + 'image': 'busybox' + } + }, 'working_dir', 'filename.yml') + ) + service_dicts = config_data.services + self.assertEqual( + service_sort(service_dicts), + service_sort([ + { + 'name': 'version', + 'image': 'busybox', + } + ]) + ) + + def test_load_invalid_version(self): + with self.assertRaises(ConfigurationError): + config.load( + build_config_details({ + 'version': 18, + 'services': { + 'foo': {'image': 'busybox'} + } + }, 'working_dir', 'filename.yml') + ) + + with self.assertRaises(ConfigurationError): + config.load( + build_config_details({ + 'version': 'two point oh', + 'services': { + 'foo': {'image': 'busybox'} + } + }, 'working_dir', 'filename.yml') + ) def test_load_throws_error_when_not_dict(self): with self.assertRaises(ConfigurationError): @@ -78,6 +157,16 @@ def test_load_throws_error_when_not_dict(self): ) ) + def test_load_throws_error_when_not_dict_v2(self): + with self.assertRaises(ConfigurationError): + config.load( + build_config_details( + {'version': 2, 'services': {'web': 'busybox:latest'}}, + 'working_dir', + 'filename.yml' + ) + ) + def test_load_config_invalid_service_names(self): for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: with pytest.raises(ConfigurationError) as exc: @@ -87,6 +176,17 @@ def test_load_config_invalid_service_names(self): 'filename.yml')) assert 'Invalid service name \'%s\'' % invalid_name in exc.exconly() + def test_config_invalid_service_names_v2(self): + for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: + with pytest.raises(ConfigurationError) as exc: + config.load( + build_config_details({ + 'version': 2, + 'services': {invalid_name: {'image': 'busybox'}} + }, 'working_dir', 'filename.yml') + ) + assert 'Invalid service name \'%s\'' % invalid_name in exc.exconly() + def test_load_with_invalid_field_name(self): config_details = build_config_details( {'web': {'image': 'busybox', 'name': 'bogus'}}, @@ -120,6 +220,22 @@ def test_config_integer_service_name_raise_validation_error(self): ) ) + def test_config_integer_service_name_raise_validation_error_v2(self): + expected_error_msg = ("In file 'filename.yml' service name: 1 needs to " + "be a string, eg '1'") + + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + config.load( + build_config_details( + { + 'version': 2, + 'services': {1: {'image': 'busybox'}} + }, + 'working_dir', + 'filename.yml' + ) + ) + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') def test_load_with_multiple_files(self): base_file = config.ConfigFile( @@ -143,7 +259,7 @@ def test_load_with_multiple_files(self): }) details = config.ConfigDetails('.', [base_file, override_file]) - service_dicts = config.load(details) + service_dicts = config.load(details).services expected = [ { 'name': 'web', @@ -170,6 +286,18 @@ def test_load_with_multiple_files_and_empty_override(self): error_msg = "Top level object in 'override.yml' needs to be an object" assert error_msg in exc.exconly() + def test_load_with_multiple_files_and_empty_override_v2(self): + base_file = config.ConfigFile( + 'base.yml', + {'version': 2, 'services': {'web': {'image': 'example/web'}}}) + override_file = config.ConfigFile('override.yml', None) + details = config.ConfigDetails('.', [base_file, override_file]) + + with pytest.raises(ConfigurationError) as exc: + config.load(details) + error_msg = "Top level object in 'override.yml' needs to be an object" + assert error_msg in exc.exconly() + def test_load_with_multiple_files_and_empty_base(self): base_file = config.ConfigFile('base.yml', None) override_file = config.ConfigFile( @@ -181,6 +309,17 @@ def test_load_with_multiple_files_and_empty_base(self): config.load(details) assert "Top level object in 'base.yml' needs to be an object" in exc.exconly() + def test_load_with_multiple_files_and_empty_base_v2(self): + base_file = config.ConfigFile('base.yml', None) + override_file = config.ConfigFile( + 'override.tml', + {'version': 2, 'services': {'web': {'image': 'example/web'}}} + ) + details = config.ConfigDetails('.', [base_file, override_file]) + with pytest.raises(ConfigurationError) as exc: + config.load(details) + assert "Top level object in 'base.yml' needs to be an object" in exc.exconly() + def test_load_with_multiple_files_and_extends_in_override_file(self): base_file = config.ConfigFile( 'base.yaml', @@ -207,7 +346,7 @@ def test_load_with_multiple_files_and_extends_in_override_file(self): labels: ['label=one'] """) with tmpdir.as_cwd(): - service_dicts = config.load(details) + service_dicts = config.load(details).services expected = [ { @@ -248,19 +387,62 @@ def test_load_sorts_in_dependency_order(self): 'volumes': ['/tmp'], } }) - services = config.load(config_details) + services = config.load(config_details).services assert services[0]['name'] == 'volume' assert services[1]['name'] == 'db' assert services[2]['name'] == 'web' + def test_load_with_multiple_files_v2(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': 2, + 'services': { + 'web': { + 'image': 'example/web', + 'links': ['db'], + }, + 'db': { + 'image': 'example/db', + } + }, + }) + override_file = config.ConfigFile( + 'override.yaml', + { + 'version': 2, + 'services': { + 'web': { + 'build': '/', + 'volumes': ['/home/user/project:/code'], + }, + } + }) + details = config.ConfigDetails('.', [base_file, override_file]) + + service_dicts = config.load(details).services + expected = [ + { + 'name': 'web', + 'build': os.path.abspath('/'), + 'links': ['db'], + 'volumes': [VolumeSpec.parse('/home/user/project:/code')], + }, + { + 'name': 'db', + 'image': 'example/db', + }, + ] + self.assertEqual(service_sort(service_dicts), service_sort(expected)) + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: services = config.load( build_config_details( {valid_name: {'image': 'busybox'}}, 'tests/fixtures/extends', - 'common.yml')) + 'common.yml')).services assert services[0]['name'] == valid_name def test_config_hint(self): @@ -451,7 +633,7 @@ def test_valid_config_which_allows_two_type_definitions(self): 'working_dir', 'filename.yml' ) - ) + ).services self.assertEqual(service[0]['expose'], expose) def test_valid_config_oneof_string_or_list(self): @@ -466,7 +648,7 @@ def test_valid_config_oneof_string_or_list(self): 'working_dir', 'filename.yml' ) - ) + ).services self.assertEqual(service[0]['entrypoint'], entrypoint) @mock.patch('compose.config.validation.log') @@ -496,7 +678,7 @@ def test_config_valid_environment_dict_key_contains_dashes(self): 'working_dir', 'filename.yml' ) - ) + ).services self.assertEqual(services[0]['environment']['SPRING_JPA_HIBERNATE_DDL-AUTO'], 'none') def test_load_yaml_with_yaml_error(self): @@ -543,7 +725,7 @@ def test_normalize_dns_options(self): 'dns_search': 'domain.local', } })) - assert actual == [ + assert actual.services == [ { 'name': 'web', 'image': 'alpine', @@ -655,7 +837,7 @@ def test_config_file_with_environment_variable(self): service_dicts = config.load( config.find('tests/fixtures/environment-interpolation', None), - ) + ).services self.assertEqual(service_dicts, [ { @@ -722,7 +904,7 @@ def test_empty_environment_key_allowed(self): '.', None, ) - )[0] + ).services[0] self.assertEquals(service_dict['environment']['POSTGRES_PASSWORD'], '') @@ -734,10 +916,14 @@ def test_no_binding(self): @mock.patch.dict(os.environ) def test_volume_binding_with_environment_variable(self): os.environ['VOLUME_PATH'] = '/host/path' - d = config.load(build_config_details( - {'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}}, - '.', - ))[0] + + d = config.load( + build_config_details( + {'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}}, + '.', + None, + ) + ).services[0] self.assertEqual(d['volumes'], [VolumeSpec.parse('/host/path:/container/path')]) @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') @@ -1012,7 +1198,7 @@ def test_validation_with_correct_memswap_values(self): 'tests/fixtures/extends', 'common.yml' ) - ) + ).services self.assertEqual(service_dict[0]['memswap_limit'], 2000000) def test_memswap_can_be_a_string(self): @@ -1022,7 +1208,7 @@ def test_memswap_can_be_a_string(self): 'tests/fixtures/extends', 'common.yml' ) - ) + ).services self.assertEqual(service_dict[0]['memswap_limit'], "512M") @@ -1126,7 +1312,7 @@ def test_resolve_path(self): {'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}}, "tests/fixtures/env", ) - )[0] + ).services[0] self.assertEqual( set(service_dict['volumes']), set([VolumeSpec.parse('/tmp:/host/tmp')])) @@ -1136,14 +1322,14 @@ def test_resolve_path(self): {'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}}, "tests/fixtures/env", ) - )[0] + ).services[0] self.assertEqual( set(service_dict['volumes']), set([VolumeSpec.parse('/opt/tmp:/opt/host/tmp')])) def load_from_filename(filename): - return config.load(config.find('.', [filename])) + return config.load(config.find('.', [filename])).services class ExtendsTest(unittest.TestCase): @@ -1313,7 +1499,7 @@ def test_extends_validation_valid_config(self): 'tests/fixtures/extends', 'common.yml' ) - ) + ).services self.assertEquals(len(service), 1) self.assertIsInstance(service[0], dict) @@ -1594,7 +1780,7 @@ def test_valid_url_in_build_path(self): for valid_url in valid_urls: service_dict = config.load(build_config_details({ 'validurl': {'build': valid_url}, - }, '.', None)) + }, '.', None)).services assert service_dict[0]['build'] == valid_url def test_invalid_url_in_build_path(self): diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index f4c6f8ca165..4bf5f463659 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -4,6 +4,7 @@ from .. import mock from .. import unittest +from compose.config.config import Config from compose.config.types import VolumeFromSpec from compose.const import LABEL_SERVICE from compose.container import Container @@ -18,7 +19,7 @@ def setUp(self): self.mock_client = mock.create_autospec(docker.Client) def test_from_dict(self): - project = Project.from_dicts('composetest', [ + project = Project.from_config('composetest', Config(None, [ { 'name': 'web', 'image': 'busybox:latest' @@ -27,7 +28,7 @@ def test_from_dict(self): 'name': 'db', 'image': 'busybox:latest' }, - ], None) + ], None), None) self.assertEqual(len(project.services), 2) self.assertEqual(project.get_service('web').name, 'web') self.assertEqual(project.get_service('web').options['image'], 'busybox:latest') @@ -35,7 +36,7 @@ def test_from_dict(self): self.assertEqual(project.get_service('db').options['image'], 'busybox:latest') def test_from_config(self): - dicts = [ + dicts = Config(None, [ { 'name': 'web', 'image': 'busybox:latest', @@ -44,8 +45,8 @@ def test_from_config(self): 'name': 'db', 'image': 'busybox:latest', }, - ] - project = Project.from_dicts('composetest', dicts, None) + ], None) + project = Project.from_config('composetest', dicts, None) self.assertEqual(len(project.services), 2) self.assertEqual(project.get_service('web').name, 'web') self.assertEqual(project.get_service('web').options['image'], 'busybox:latest') @@ -141,13 +142,13 @@ def test_use_volumes_from_container(self): container_id = 'aabbccddee' container_dict = dict(Name='aaa', Id=container_id) self.mock_client.inspect_container.return_value = container_dict - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'test', 'image': 'busybox:latest', 'volumes_from': [VolumeFromSpec('aaa', 'rw')] } - ], self.mock_client) + ], None), self.mock_client) self.assertEqual(project.get_service('test')._get_volumes_from(), [container_id + ":rw"]) def test_use_volumes_from_service_no_container(self): @@ -160,7 +161,7 @@ def test_use_volumes_from_service_no_container(self): "Image": 'busybox:latest' } ] - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'vol', 'image': 'busybox:latest' @@ -170,13 +171,13 @@ def test_use_volumes_from_service_no_container(self): 'image': 'busybox:latest', 'volumes_from': [VolumeFromSpec('vol', 'rw')] } - ], self.mock_client) + ], None), self.mock_client) self.assertEqual(project.get_service('test')._get_volumes_from(), [container_name + ":rw"]) def test_use_volumes_from_service_container(self): container_ids = ['aabbccddee', '12345'] - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'vol', 'image': 'busybox:latest' @@ -186,7 +187,7 @@ def test_use_volumes_from_service_container(self): 'image': 'busybox:latest', 'volumes_from': [VolumeFromSpec('vol', 'rw')] } - ], None) + ], None), None) with mock.patch.object(Service, 'containers') as mock_return: mock_return.return_value = [ mock.Mock(id=container_id, spec=Container) @@ -196,12 +197,12 @@ def test_use_volumes_from_service_container(self): [container_ids[0] + ':rw']) def test_net_unset(self): - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'test', 'image': 'busybox:latest', } - ], self.mock_client) + ], None), self.mock_client) service = project.get_service('test') self.assertEqual(service.net.id, None) self.assertNotIn('NetworkMode', service._get_container_host_config({})) @@ -210,13 +211,13 @@ def test_use_net_from_container(self): container_id = 'aabbccddee' container_dict = dict(Name='aaa', Id=container_id) self.mock_client.inspect_container.return_value = container_dict - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'test', 'image': 'busybox:latest', 'net': 'container:aaa' } - ], self.mock_client) + ], None), self.mock_client) service = project.get_service('test') self.assertEqual(service.net.mode, 'container:' + container_id) @@ -230,7 +231,7 @@ def test_use_net_from_service(self): "Image": 'busybox:latest' } ] - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'aaa', 'image': 'busybox:latest' @@ -240,7 +241,7 @@ def test_use_net_from_service(self): 'image': 'busybox:latest', 'net': 'container:aaa' } - ], self.mock_client) + ], None), self.mock_client) service = project.get_service('test') self.assertEqual(service.net.mode, 'container:' + container_name) @@ -285,12 +286,12 @@ def test_container_without_name(self): }, }, } - project = Project.from_dicts( + project = Project.from_config( 'test', - [{ + Config(None, [{ 'name': 'web', 'image': 'busybox:latest', - }], + }], None), self.mock_client, ) self.assertEqual([c.id for c in project.containers()], ['1']) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 1c8b441f343..87d6af59550 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -234,6 +234,7 @@ def test_get_container_create_options_does_not_mutate_options(self): prev_container = mock.Mock( id='ababab', image_config={'ContainerConfig': {}}) + prev_container.get.return_value = None opts = service._get_container_create_options( {}, @@ -575,6 +576,10 @@ def test_net_service_no_containers(self): self.assertEqual(net.service_name, service_name) +def build_mount(destination, source, mode='rw'): + return {'Source': source, 'Destination': destination, 'Mode': mode} + + class ServiceVolumesTest(unittest.TestCase): def setUp(self): @@ -600,12 +605,33 @@ def test_get_container_data_volumes(self): } container = Container(self.mock_client, { 'Image': 'ababab', - 'Volumes': { - '/host/volume': '/host/volume', - '/existing/volume': '/var/lib/docker/aaaaaaaa', - '/removed/volume': '/var/lib/docker/bbbbbbbb', - '/mnt/image/data': '/var/lib/docker/cccccccc', - }, + 'Mounts': [ + { + 'Source': '/host/volume', + 'Destination': '/host/volume', + 'Mode': '', + 'RW': True, + 'Name': 'hostvolume', + }, { + 'Source': '/var/lib/docker/aaaaaaaa', + 'Destination': '/existing/volume', + 'Mode': '', + 'RW': True, + 'Name': 'existingvolume', + }, { + 'Source': '/var/lib/docker/bbbbbbbb', + 'Destination': '/removed/volume', + 'Mode': '', + 'RW': True, + 'Name': 'removedvolume', + }, { + 'Source': '/var/lib/docker/cccccccc', + 'Destination': '/mnt/image/data', + 'Mode': '', + 'RW': True, + 'Name': 'imagedata', + }, + ] }, has_been_inspected=True) expected = [ @@ -630,7 +656,13 @@ def test_merge_volume_bindings(self): intermediate_container = Container(self.mock_client, { 'Image': 'ababab', - 'Volumes': {'/existing/volume': '/var/lib/docker/aaaaaaaa'}, + 'Mounts': [{ + 'Source': '/var/lib/docker/aaaaaaaa', + 'Destination': '/existing/volume', + 'Mode': '', + 'RW': True, + 'Name': 'existingvolume', + }], }, has_been_inspected=True) expected = [ @@ -693,9 +725,16 @@ def test_different_host_path_in_container_json(self): self.mock_client.inspect_container.return_value = { 'Id': '123123123', 'Image': 'ababab', - 'Volumes': { - '/data': '/mnt/sda1/host/path', - }, + 'Mounts': [ + { + 'Destination': '/data', + 'Source': '/mnt/sda1/host/path', + 'Mode': '', + 'RW': True, + 'Driver': 'local', + 'Name': 'abcdefff1234' + }, + ] } service._get_container_create_options(