[Backport] Multiple devstack releases with one machine

Makefile.edx

@@ -23,12 +23,12 @@ dev.provision.whitelabel:
 # The containers must be started with the 'dev.up.e2e_wl_tests' target.
 # AND the test must be setup using the 'dev.provision.whitelabel' target.
 whitelabel-tests:
-	docker run -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e
-	docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh devstack.whitelabel:/tmp/run_whitelabel_tests.sh
-	docker exec -t devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh
+	docker-compose run -T -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e
+	docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh $(make --silent dev.print-container.devstack.whitelabel)":/tmp/run_whitelabel_tests.sh
+	docker-compose $(DOCKER_COMPOSE_FILES) exec devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh
 
 whitelabel-cleanup:
-	docker rm --force devstack.whitelabel
+	docker-compose rm --force devstack.whitelabel
 
 whitelabel-shell: ## Start a whitelabel test shell session
-	docker exec -it devstack.whitelabel env TERM=$(TERM) bash
+	docker-compose $(DOCKER_COMPOSE_FILES) exec -it devstack.whitelabel env TERM=$(TERM) bash

README.rst

@@ -404,6 +404,177 @@ images until you change or unset ``OPENEDX_RELEASE`` again.  To work on the
 master branches and ``latest`` images, unset ``OPENEDX_RELEASE`` or set it to
 an empty string.
 
+How do I run multiple named Open edX releases on same machine?
+--------------------------------------------------------------
+You can have multiple isolated Devstacks provisioned on a single computer now. Follow these directions to switch between the named releases.
+
+#. Bring down any running containers by issuing a `make stop.all`. 
+#. The ``COMPOSE_PROJECT_NAME`` variable is used to define Docker namespaced volumes and network based on this value, so changing it will give you a separate set of databases. This is handled for you automatically by setting the ``OPENEDX_RELEASE`` environment variable in ``options.mk`` (e.g. ``COMPOSE_PROJECT_NAME=devstack-juniper.master``. Should you want to manually override this edit the ``options.local.mk`` in the root of this repo and create the file if it does not exist. Change the devstack project name by adding the following line:
+   ``COMPOSE_PROJECT_NAME=<your-alternate-devstack-name>`` (e.g. ``COMPOSE_PROJECT_NAME=secondarydevstack``)
+#. Perform steps in `How do I run the images for a named Open edX release?`_ for specific release.
+#. Follow the steps in `Getting Started`_ section to update requirements (e.g. ``make requirements``) and provision (e.g. ``make dev.provision``) the new named release containers.
+
+As a specific example, if ``OPENEDX_RELEASE`` is set in your environment as ``juniper.master``, then ``COMPOSE_PROJECT_NAME`` will default to ``devstack-juniper.master`` instead of ``devstack``.
+
+The implication of this is that you can switch between isolated Devstack databases by changing the value of the ``OPENEDX_RELEASE`` environment variable.
+
+Switch between your Devstack releases by doing the following:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+#. Bring down the containers by issuing a ``make stop.all`` for the running release.
+#. Follow the instructions from the `How do I run multiple named Open edX releases on same machine?`_ section.
+#. Edit the project name in ``options.local.mk`` or set the ``OPENEDX_RELEASE`` environment variable and let the ``COMPOSE_PROJECT_NAME`` be assigned automatically. 
+#. Bring up the containers with ``make dev.up``.
+
+**NOTE:** Additional instructions on switching releases using `direnv` can be found in `How do I switch releases using 'direnv'?`_ section.
+
+Examples of Docker Service Names After Setting the ``COMPOSE_PROJECT_NAME`` variable. Notice that the **devstack-juniper.master** name represents the ``COMPOSE_PROJECT_NAME``.
+
+-  edx.devstack-juniper.master.lms          
+-  edx.devstack-juniper.master.mysql  
+
+Each instance has an isolated set of databases. This could, for example, be used to quickly switch between versions of Open edX without hitting as many issues with migrations, data integrity, etc.
+
+Unfortunately, this **does not** currently support running Devstacks simultaneously, because we hard-code host port numbers all over the place, and two running containers cannot share the same host port.
+
+Questions & Troubleshooting – Multiple Named Open edX Releases on Same Machine
+------------------------------------------------------------------------------
+
+This broke my existing Devstack!
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ See if the troubleshooting of this readme can help resolve your broken devstack first, then try posting on the `Open edX forums <https://discuss.openedx.org>`__ to see if you have the same issue as any others. If you think you have found a bug, file a CR ticket.
+
+I’m getting errors related to ports already being used.
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Make sure you bring down your devstack before changing the value of COMPOSE_PROJECT_NAME. If you forgot to, change the COMPOSE_PROJECT_NAME back to its original value, run ``make dev.down``, and then try again.
+
+I have custom scripts/compose files that integrate with or extend Devstack. Will those still work?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+With the default value of COMPOSE_PROJECT_NAME = devstack, they should still work. If you choose a different COMPOSE_PROJECT_NAME, your extensions will likely break, because the names of containers change along with the project name.
+
+How do I switch releases using 'direnv'?
+----------------------------------------
+
+Follow directions in `Switch between your Devstack releases by doing the following:`_ then make the following adjustments.
+
+Make sure that you have setup each Open edX release in separate directories using `How do I enable environment variables for current directory using 'direnv'?`_ instructions. Open the next release project in a separate code editor, then activate the ``direnv`` environment variables and virtual environment for the next release by using a terminal shell to traverse to the directory with the corresponding release ``.envrc`` file. You may need to issue a ``direnv allow`` command to enable the ``.envrc`` file.
+
+    .. code:: sh
+
+        # You should see something like the following after successfully enabling 'direnv' for the Juniper release.
+
+        direnv: loading ~/open-edx/devstack.juniper/.envrc   
+        direnv: export +DEVSTACK_WORKSPACE +OPENEDX_RELEASE +VIRTUAL_ENV ~PATH
+        (venv)username@computer-name devstack.juniper %
+
+**NOTE:** Setting of the ``OPENEDX_RELEASE`` should have been handled within the ``.envrc`` file for named releases only and should not be defined for the ``master`` release.
+
+How do I enable environment variables for current directory using 'direnv'?
+---------------------------------------------------------------------------
+We recommend separating the named releases into different directories, for clarity purposes. You can use `direnv <https://direnv.net/>`__ to define different environment variables per directory::
+
+    .. code::
+
+        # Example showing directory structure for separate Open edX releases.
+
+        /Users/<username>/open-edx – root directory for platform development
+        |_ ./devstack.master  – directory containing all repository information related to the main development release.
+        |_ ./devstack.juniper – directory containing all repository information related to the Juniper release.
+
+#. Install `direnv` using instructions on https://direnv.net/. Below you will find additional setup at the time of this writing so refer to latest of `direnv` site for additional configuration needed.
+
+#. Setup the following configuration to hook `direnv` for local directory environment overrides. There are two examples for BASH or ZSH (Mac OS X) shells.
+
+    .. code:: sh
+
+        ## ~/.bashrc for BASH shell
+
+        ## Hook in `direnv` for local directory environment overrides.
+        ## https://direnv.net/docs/hook.html
+        eval "$(direnv hook bash)"
+
+        # https://github.com/direnv/direnv/wiki/Python#bash
+        show_virtual_env() {
+        if [[ -n "$VIRTUAL_ENV" && -n "$DIRENV_DIR" ]]; then
+            echo "($(basename $VIRTUAL_ENV))"
+        fi
+        }
+        export -f show_virtual_env
+        PS1='$(show_virtual_env)'$PS1
+
+        # ---------------------------------------------------
+
+        ## ~/.zshrc for ZSH shell for Mac OS X.
+
+        ## Hook in `direnv` for local directory environment setup.
+        ## https://direnv.net/docs/hook.html 
+        eval "$(direnv hook zsh)"
+
+        # https://github.com/direnv/direnv/wiki/Python#zsh
+        setopt PROMPT_SUBST
+
+        show_virtual_env() {
+        if [[ -n "$VIRTUAL_ENV" && -n "$DIRENV_DIR" ]]; then
+            echo "($(basename $VIRTUAL_ENV))"
+        fi
+        }
+        PS1='$(show_virtual_env)'$PS1
+
+#. Setup `layout_python-venv` function to be used in local project directory `.envrc` file.
+
+    .. code:: sh
+
+        ## ~/.config/direnv/direnvrc
+
+        # https://github.com/direnv/direnv/wiki/Python#venv-stdlib-module
+
+        realpath() {
+            [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"
+        }
+        layout_python-venv() {
+            local python=${1:-python3}
+            [[ $# -gt 0 ]] && shift
+            unset PYTHONHOME
+            if [[ -n $VIRTUAL_ENV ]]; then
+                VIRTUAL_ENV=$(realpath "${VIRTUAL_ENV}")
+            else
+                local python_version
+                python_version=$("$python" -c "import platform; print(platform.python_version())")
+                if [[ -z $python_version ]]; then
+                    log_error "Could not detect Python version"
+                    return 1
+                fi
+                VIRTUAL_ENV=$PWD/.direnv/python-venv-$python_version
+            fi
+            export VIRTUAL_ENV
+            if [[ ! -d $VIRTUAL_ENV ]]; then
+                log_status "no venv found; creating $VIRTUAL_ENV"
+                "$python" -m venv "$VIRTUAL_ENV"
+            fi
+
+            PATH="${VIRTUAL_ENV}/bin:${PATH}"
+            export PATH
+        }
+
+#. Example `.envrc` file used in project directory. Need to make sure that each release root has this unique file. 
+
+    .. code:: sh
+
+        # Open edX named release project directory root.
+        ## <project-path>/devstack.juniper/.envrc
+
+        # https://discuss.openedx.org/t/docker-devstack-multiple-releases-one-machine/1902/10
+
+        # This is handled when OPENEDX_RELEASE is set. Leaving this in for manual override.
+        # export COMPOSE_PROJECT_NAME=devstack-juniper
+
+        export DEVSTACK_WORKSPACE="$(pwd)"
+        export OPENEDX_RELEASE=juniper.master
+        export VIRTUAL_ENV="$(pwd)/devstack/venv"
+
+        # https://github.com/direnv/direnv/wiki/Python#virtualenv
+        layout python-venv
+
 How do I create database dumps?
 -------------------------------
 We use database dumps to speed up provisioning and generally spend less time running migrations. These dumps should be

course-generator/create-courses.sh

@@ -8,21 +8,21 @@ echo "Parsing options"
 container_error=false
 for arg in "$@"; do
     if [ $arg == "--studio" ]; then
-        if [ ! "$(docker-compose exec lms bash -c 'echo "Course will be created for studio"; exit $?')" ]; then
+        if [ ! "$(docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'echo "Course will be created for studio"; exit $?')" ]; then
             echo "Issue with studio container"
             container_error=true
         else
             studio=true
         fi
     elif [ $arg == "--ecommerce" ]; then
-        if [ ! "$(docker exec -t edx.devstack.ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then
+        if [ ! "$(docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then
             echo "Issue with ecommerce container"
             container_error=true
         else
             ecommerce=true
         fi
     elif [ $arg == "--marketing" ]; then
-        if [ ! "$(docker exec -t edx.devstack.marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then
+        if [ ! "$(docker-compose $DOCKER_COMPOSE_FILES exec marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then
             echo "Issue with marketing container. Course creation will proceed without marketing container."
         else
             marketing=true
@@ -49,15 +49,15 @@ done < "${@: -1}"
 
 if $studio ; then
 	echo "Creating courses on studio."
-	docker-compose exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'"
+	docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'"
 fi
 
 if $ecommerce ; then
 	echo "Creating courses on ecommerce."
-	docker exec -t edx.devstack.ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'"
+	docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'"
 fi
 
 if $marketing ; then
 	echo "Creating courses on marketing."
-	docker exec -t edx.devstack.marketing bash -c "drush generate_courses '$course_json'"
+	docker-compose $DOCKER_COMPOSE_FILES exec marketing bash -c "drush generate_courses '$course_json'"
 fi

docker-compose-analytics-pipeline.yml

@@ -2,34 +2,42 @@ version: "2.1"
 
 services:
   namenode:
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.namenode"
     image: edxops/analytics_pipeline_hadoop_namenode:${OPENEDX_RELEASE:-latest}
-    container_name: edx.devstack.analytics_pipeline.namenode
     hostname: namenode
     environment:
       - CLUSTER_NAME=devstack
+    networks:
+      default:
+        aliases:
+          - edx.devstack.analytics_pipeline.namenode
     ports:
       - 127.0.0.1:50070:50070
     command: ["/run.sh"]
     volumes:
       - namenode_data:/hadoop/dfs/name
 
   datanode:
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.datanode"
     image: edxops/analytics_pipeline_hadoop_datanode:${OPENEDX_RELEASE:-latest}
-    container_name: edx.devstack.analytics_pipeline.datanode
     hostname: datanode
     environment:
       CORE_CONF_fs_defaultFS: "hdfs://namenode:8020"
     depends_on:
       - namenode
+    networks:
+      default:
+        aliases:
+          - edx.devstack.analytics_pipeline.datanode
     ports:
       - 127.0.0.1:50075:50075
     command: ["/run.sh"]
     volumes:
       - datanode_data:/hadoop/dfs/data
 
   resourcemanager:
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.resourcemanager"
     image: edxops/analytics_pipeline_hadoop_resourcemanager:${OPENEDX_RELEASE:-latest}
-    container_name: edx.devstack.analytics_pipeline.resourcemanager
     hostname: resourcemanager
     environment:
       CORE_CONF_fs_defaultFS: "hdfs://namenode:8020"
@@ -40,13 +48,17 @@ services:
     depends_on:
       - namenode
       - datanode
+    networks:
+      default:
+        aliases:
+          - edx.devstack.analytics_pipeline.resourcemanager
     ports:
       - 127.0.0.1:8088:8088      # resource manager web ui
     command: ["/run.sh"]
 
   nodemanager:
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.nodemanager"
     image: edxops/analytics_pipeline_hadoop_nodemanager:${OPENEDX_RELEASE:-latest}
-    container_name: edx.devstack.analytics_pipeline.nodemanager
     hostname: nodemanager
     environment:
       CORE_CONF_fs_defaultFS: "hdfs://namenode:8020"
@@ -60,41 +72,58 @@ services:
       - resourcemanager
       - namenode
       - datanode
+    networks:
+      default:
+        aliases:
+          - edx.devstack.analytics_pipeline.nodemanager
     ports:
       - 127.0.0.1:8042:8042      # node manager web ui
       - 127.0.0.1:19888:19888    # node manager job history server ui
     command: ["/run.sh"]
 
   sparkmaster:
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkmaster"
     image: edxops/analytics_pipeline_spark_master:${OPENEDX_RELEASE:-latest}
-    container_name: edx.devstack.analytics_pipeline.sparkmaster
     hostname: sparkmaster
+    networks:
+      default:
+        aliases:
+          - edx.devstack.analytics_pipeline.sparkmaster
     ports:
       - 127.0.0.1:8080:8080
       - 127.0.0.1:7077:7077       # spark master port
       - 127.0.0.1:6066:6066       # spark api
       - 127.0.0.1:18080:18080     # spark history server
 
   sparkworker:
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkworker"
     image: edxops/analytics_pipeline_spark_worker:${OPENEDX_RELEASE:-latest}
-    container_name: edx.devstack.analytics_pipeline.sparkworker
     hostname: sparkworker
     depends_on:
       - sparkmaster
     environment:
       - SPARK_MASTER=spark://sparkmaster:7077
+    networks:
+      default:
+        aliases:
+          - edx.devstack.analytics_pipeline.sparkworker
     ports:
       - 127.0.0.1:8081:8081       # spark worker UI
 
   vertica:
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.vertica"
     image: sumitchawla/vertica:latest
-    container_name: edx.devstack.analytics_pipeline.vertica
+    hostname: vertica
+    networks:
+      default:
+        aliases:
+          - edx.devstack.analytics_pipeline.vertica
     volumes:
       - vertica_data:/home/dbadmin/docker
 
   analyticspipeline:
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline"
     image: edxops/analytics_pipeline:${OPENEDX_RELEASE:-latest}
-    container_name: edx.devstack.analytics_pipeline
     hostname: analyticspipeline
     volumes:
       - ${DEVSTACK_WORKSPACE}/edx-analytics-pipeline:/edx/app/analytics_pipeline/analytics_pipeline
@@ -108,6 +137,10 @@ services:
       - sparkworker
       - elasticsearch
       - vertica
+    networks:
+      default:
+        aliases:
+          - edx.devstack.analytics_pipeline
     ports:
       - 127.0.0.1:4040:4040     # spark web UI
     environment:

docker-compose-marketing-site.yml

@@ -7,7 +7,7 @@ services:
       - MARKETING_SITE_ROOT="http://localhost:8080"
 
   marketing:
-    container_name: edx.devstack.marketing
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.marketing"
     depends_on:
       - mysql
       - memcached
@@ -26,5 +26,9 @@ services:
       # IP address of your machine to enable debugging (IP_ADDRESS set in .env file)
       - XDEBUG_CONFIG=remote_host=${XDEBUG_IP_ADDRESS:-127.0.0.1}
     image: edxops/edx-mktg:${OPENEDX_RELEASE:-latest}
+    networks:
+      default:
+        aliases:
+          - edx.devstack.marketing
     ports:
       - "8080:80"