diff --git a/Makefile b/Makefile index 1134c14c5e..6e2bf4ac77 100644 --- a/Makefile +++ b/Makefile @@ -177,6 +177,10 @@ dev.nfs.%: dev.up.all: dev.up dev.up.watchers ## Bring up all services with host volumes, including watchers +dev.print-container.%: ## Get the ID of the running container for a given service. + @# Can be run as ``make --silent dev.print-container.$service`` for just ID. + @echo $$(docker-compose $(DOCKER_COMPOSE_FILES) ps --quiet $*) + # TODO: Improve or rip out Docker Sync targets. # They are not well-fleshed-out and it is not clear if anyone uses them. @@ -202,7 +206,7 @@ provision: | dev.provision ## This command will be deprecated in a future releas stop: ## Stop all services (test -d .docker-sync && docker-sync stop) || true ## Ignore failure here - docker-compose stop + docker-compose $(DOCKER_COMPOSE_FILES) stop stop.watchers: ## Stop asset watchers docker-compose $(DOCKER_COMPOSE_FILES) stop lms_watcher studio_watcher @@ -263,96 +267,102 @@ pull: dev.pull pull.xqueue: dev.pull.without-deps.xqueue+xqueue_consumer validate: ## Validate the devstack configuration - docker-compose config + docker-compose $(DOCKER_COMPOSE_FILES) config -backup: dev.up.mysql+mongo+elasticsearch ## Write all data volumes to the host. - docker run --rm --volumes-from edx.devstack.mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql - docker run --rm --volumes-from edx.devstack.mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db - docker run --rm --volumes-from edx.devstack.elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data +backup: ## Write all data volumes to the host. + docker run --rm --volumes-from -T $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql + docker run --rm --volumes-from -T $$(make -s dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db + docker run --rm --volumes-from -T $$(make -s dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data -restore: dev.up.mysql+mongo+elasticsearch ## Restore all data volumes from the host. WARNING: THIS WILL OVERWRITE ALL EXISTING DATA! - docker run --rm --volumes-from edx.devstack.mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz - docker run --rm --volumes-from edx.devstack.mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz - docker run --rm --volumes-from edx.devstack.elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz +restore: ## Restore all data volumes from the host. WARNING: THIS WILL OVERWRITE ALL EXISTING DATA! + docker run --rm --volumes-from -T $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz + docker run --rm --volumes-from -T $$(make -s dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz + docker run --rm --volumes-from -T $$(make -s dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz # TODO: Print out help for this target. Even better if we can iterate over the # services in docker-compose.yml, and print the actual service names. %-shell: ## Run a shell on the specified service container - docker exec -it edx.devstack.$* /bin/bash + docker-compose $(DOCKER_COMPOSE_FILES) exec $* /bin/bash analyticspipeline-shell: ## Run a shell on the analytics pipeline container - docker exec -it edx.devstack.analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open credentials-shell: ## Run a shell on the credentials container - docker exec -it edx.devstack.credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' + docker-compose $(DOCKER_COMPOSE_FILES) exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' discovery-shell: ## Run a shell on the discovery container - docker exec -it edx.devstack.discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open ecommerce-shell: ## Run a shell on the ecommerce container - docker exec -it edx.devstack.ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open e2e-shell: ## Start the end-to-end tests container with a shell - docker run -it --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash + docker-compose run --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash registrar-shell: ## Run a shell on the registrar site container - docker exec -it edx.devstack.registrar env TERM=$(TERM) /edx/app/registrar/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec registrar env TERM=$(TERM) /edx/app/registrar/devstack.sh open %-update-db: ## Run migrations for the specified service container - docker exec -t edx.devstack.$* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' + docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' studio-update-db: ## Run migrations for the Studio container - docker exec -t edx.devstack.studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' + docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' lms-update-db: ## Run migrations LMS container - docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' + docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' update-db: | $(DB_MIGRATION_TARGETS) ## Run the migrations for DEFAULT_SERVICES forum-restart-devserver: ## Kill the forum's Sinatra development server. The watcher process will restart it. - docker exec -t edx.devstack.forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose $(DOCKER_COMPOSE_FILES) forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")' lms-shell: ## Run a shell on the LMS container - docker exec -it edx.devstack.lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open lms-watcher-shell: ## Run a shell on the LMS watcher container - docker exec -it edx.devstack.lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open %-attach: ## Attach to the specified service container process to use the debugger & see logs. - docker attach edx.devstack.$* + docker attach "$$(make --silent dev.print-container.$*)" lms-restart: lms-restart-devserver studio-shell: ## Run a shell on the Studio container - docker exec -it edx.devstack.studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-watcher-shell: ## Run a shell on the studio watcher container - docker exec -it edx.devstack.studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-restart: studio-restart-devserver xqueue-shell: ## Run a shell on the XQueue container - docker exec -it edx.devstack.xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue-restart: xqueue-restart-devserver %-restart-devserver: ## Kill a service's Django development server. The watcher process should restart it. - docker exec -t edx.devstack.$* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' xqueue_consumer-shell: ## Run a shell on the XQueue consumer container - docker exec -it edx.devstack.xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue_consumer-restart: ## Kill the XQueue development server. The watcher process will restart it. - docker exec -t edx.devstack.xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")' %-static: ## Rebuild static assets for the specified service container - docker exec -t edx.devstack.$* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' + docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' + +lms-sass: ## Rebuild sass assets for the LMS container + docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=lms --theme-dirs /edx/app/edx-themes/edx-platform/' lms-static: ## Rebuild static assets for the LMS container - docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets lms' + docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/' + +studio-sass: ## Rebuild sass assets for the Studio container + docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=cms --theme-dirs /edx/app/edx-themes/edx-platform/' studio-static: ## Rebuild static assets for the Studio container - docker exec -t edx.devstack.studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets studio' + docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/' static: | credentials-static discovery-static ecommerce-static lms-static studio-static ## Rebuild static assets for all service containers @@ -365,24 +375,24 @@ e2e-tests: ## Run the end-to-end tests against the service containers validate-lms-volume: ## Validate that changes to the local workspace are reflected in the LMS container touch $(DEVSTACK_WORKSPACE)/edx-platform/testfile - docker exec edx.devstack.lms ls /edx/app/edxapp/edx-platform/testfile + docker-compose $(DOCKER_COMPOSE_FILES) exec -T lms ls /edx/app/edxapp/edx-platform/testfile rm $(DEVSTACK_WORKSPACE)/edx-platform/testfile vnc-passwords: ## Get the VNC passwords for the Chrome and Firefox Selenium containers - @docker logs edx.devstack.chrome 2>&1 | grep "VNC password" | tail -1 - @docker logs edx.devstack.firefox 2>&1 | grep "VNC password" | tail -1 + @docker-compose $(DOCKER_COMPOSE_FILES) logs chrome 2>&1 | grep "VNC password" | tail -1 + @docker-compose $(DOCKER_COMPOSE_FILES) logs firefox 2>&1 | grep "VNC password" | tail -1 devpi-password: ## Get the root devpi password for the devpi container - docker-compose exec devpi bash -c "cat /data/server/.serverpassword" + docker-compose $(DOCKER_COMPOSE_FILES) exec devpi bash -c "cat /data/server/.serverpassword" mysql-shell: ## Run a shell on the mysql container - docker-compose exec mysql bash + docker-compose $(DOCKER_COMPOSE_FILES) exec mysql bash mysql-shell-edxapp: ## Run a mysql shell on the edxapp database - docker-compose exec mysql bash -c "mysql edxapp" + docker-compose $(DOCKER_COMPOSE_FILES) exec mysql bash -c "mysql edxapp" mongo-shell: ## Run a shell on the mongo container - docker-compose exec mongo bash + docker-compose $(DOCKER_COMPOSE_FILES) exec mongo bash dev.provision.analytics_pipeline: dev.provision.services.analyticspipeline @@ -393,7 +403,7 @@ dev.up.analytics_pipeline: dev.up.analyticspipeline ## Bring up analytics pipeli pull.analytics_pipeline: dev.pull.analyticspipeline ## Update analytics pipeline docker images analytics-pipeline-devstack-test: ## Run analytics pipeline tests in travis build - docker exec -u hadoop -i edx.devstack.analytics_pipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity' + docker-compose $(DOCKER_COMPOSE_FILES) exec -u hadoop -T analytics_pipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity' stop.analytics_pipeline: ## Stop all Analytics pipeline services. docker-compose $(DOCKER_COMPOSE_FILES) stop \ @@ -401,7 +411,7 @@ stop.analytics_pipeline: ## Stop all Analytics pipeline services. sparkworker vertica analyticspipeline hadoop-application-logs-%: ## View hadoop logs by application Id - docker exec -it edx.devstack.analytics_pipeline.nodemanager yarn logs -applicationId $* + docker-compose $(DOCKER_COMPOSE_FILES) exec analytics_pipeline.nodemanager yarn logs -applicationId $* # Provisions studio, ecommerce, and marketing with course(s) in test-course.json # Modify test-course.json before running this make target to generate a custom course diff --git a/Makefile.edx b/Makefile.edx index 81343ba2de..3a32bbe319 100644 --- a/Makefile.edx +++ b/Makefile.edx @@ -23,12 +23,12 @@ dev.provision.whitelabel: # The containers must be started with the 'dev.up.e2e_wl_tests' target. # AND the test must be setup using the 'dev.provision.whitelabel' target. whitelabel-tests: - docker run -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e - docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh devstack.whitelabel:/tmp/run_whitelabel_tests.sh - docker exec -t devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh + docker-compose run -T -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e + docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh $(make --silent dev.print-container.devstack.whitelabel)":/tmp/run_whitelabel_tests.sh + docker-compose $(DOCKER_COMPOSE_FILES) exec devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh whitelabel-cleanup: - docker rm --force devstack.whitelabel + docker-compose rm --force devstack.whitelabel whitelabel-shell: ## Start a whitelabel test shell session - docker exec -it devstack.whitelabel env TERM=$(TERM) bash + docker-compose $(DOCKER_COMPOSE_FILES) exec -it devstack.whitelabel env TERM=$(TERM) bash diff --git a/README.rst b/README.rst index d57bf7653a..63a9e337f0 100644 --- a/README.rst +++ b/README.rst @@ -152,7 +152,7 @@ All of the services can be run by following the steps below. For analyticstack, make dev.sync.provision - Provision using `nfs`_: + Provision using NFS: .. code:: sh @@ -177,7 +177,7 @@ All of the services can be run by following the steps below. For analyticstack, make dev.sync.up - Start using `nfs`_: + Start using NFS: .. code:: sh @@ -538,6 +538,177 @@ images until you change or unset ``OPENEDX_RELEASE`` again. To work on the master branches and ``latest`` images, unset ``OPENEDX_RELEASE`` or set it to an empty string. +How do I run multiple named Open edX releases on same machine? +-------------------------------------------------------------- +You can have multiple isolated Devstacks provisioned on a single computer now. Follow these directions to switch between the named releases. + +#. Bring down any running containers by issuing a `make stop.all`. +#. The ``COMPOSE_PROJECT_NAME`` variable is used to define Docker namespaced volumes and network based on this value, so changing it will give you a separate set of databases. This is handled for you automatically by setting the ``OPENEDX_RELEASE`` environment variable in ``options.mk`` (e.g. ``COMPOSE_PROJECT_NAME=devstack-juniper.master``. Should you want to manually override this edit the ``options.local.mk`` in the root of this repo and create the file if it does not exist. Change the devstack project name by adding the following line: + ``COMPOSE_PROJECT_NAME=`` (e.g. ``COMPOSE_PROJECT_NAME=secondarydevstack``) +#. Perform steps in `How do I run the images for a named Open edX release?`_ for specific release. +#. Follow the steps in `Getting Started`_ section to update requirements (e.g. ``make requirements``) and provision (e.g. ``make dev.provision``) the new named release containers. + +As a specific example, if ``OPENEDX_RELEASE`` is set in your environment as ``juniper.master``, then ``COMPOSE_PROJECT_NAME`` will default to ``devstack-juniper.master`` instead of ``devstack``. + +The implication of this is that you can switch between isolated Devstack databases by changing the value of the ``OPENEDX_RELEASE`` environment variable. + +Switch between your Devstack releases by doing the following: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +#. Bring down the containers by issuing a ``make stop.all`` for the running release. +#. Follow the instructions from the `How do I run multiple named Open edX releases on same machine?`_ section. +#. Edit the project name in ``options.local.mk`` or set the ``OPENEDX_RELEASE`` environment variable and let the ``COMPOSE_PROJECT_NAME`` be assigned automatically. +#. Bring up the containers with ``make dev.up``. + +**NOTE:** Additional instructions on switching releases using `direnv` can be found in `How do I switch releases using 'direnv'?`_ section. + +Examples of Docker Service Names After Setting the ``COMPOSE_PROJECT_NAME`` variable. Notice that the **devstack-juniper.master** name represents the ``COMPOSE_PROJECT_NAME``. + +- edx.devstack-juniper.master.lms +- edx.devstack-juniper.master.mysql + +Each instance has an isolated set of databases. This could, for example, be used to quickly switch between versions of Open edX without hitting as many issues with migrations, data integrity, etc. + +Unfortunately, this **does not** currently support running Devstacks simultaneously, because we hard-code host port numbers all over the place, and two running containers cannot share the same host port. + +Questions & Troubleshooting – Multiple Named Open edX Releases on Same Machine +------------------------------------------------------------------------------ + +This broke my existing Devstack! +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + See if the troubleshooting of this readme can help resolve your broken devstack first, then try posting on the `Open edX forums `__ to see if you have the same issue as any others. If you think you have found a bug, file a CR ticket. + +I’m getting errors related to ports already being used. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Make sure you bring down your devstack before changing the value of COMPOSE_PROJECT_NAME. If you forgot to, change the COMPOSE_PROJECT_NAME back to its original value, run ``make dev.down``, and then try again. + +I have custom scripts/compose files that integrate with or extend Devstack. Will those still work? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +With the default value of COMPOSE_PROJECT_NAME = devstack, they should still work. If you choose a different COMPOSE_PROJECT_NAME, your extensions will likely break, because the names of containers change along with the project name. + +How do I switch releases using 'direnv'? +---------------------------------------- + +Follow directions in `Switch between your Devstack releases by doing the following:`_ then make the following adjustments. + +Make sure that you have setup each Open edX release in separate directories using `How do I enable environment variables for current directory using 'direnv'?`_ instructions. Open the next release project in a separate code editor, then activate the ``direnv`` environment variables and virtual environment for the next release by using a terminal shell to traverse to the directory with the corresponding release ``.envrc`` file. You may need to issue a ``direnv allow`` command to enable the ``.envrc`` file. + + .. code:: sh + + # You should see something like the following after successfully enabling 'direnv' for the Juniper release. + + direnv: loading ~/open-edx/devstack.juniper/.envrc + direnv: export +DEVSTACK_WORKSPACE +OPENEDX_RELEASE +VIRTUAL_ENV ~PATH + (venv)username@computer-name devstack.juniper % + +**NOTE:** Setting of the ``OPENEDX_RELEASE`` should have been handled within the ``.envrc`` file for named releases only and should not be defined for the ``master`` release. + +How do I enable environment variables for current directory using 'direnv'? +--------------------------------------------------------------------------- +We recommend separating the named releases into different directories, for clarity purposes. You can use `direnv `__ to define different environment variables per directory:: + + .. code:: + + # Example showing directory structure for separate Open edX releases. + + /Users//open-edx – root directory for platform development + |_ ./devstack.master – directory containing all repository information related to the main development release. + |_ ./devstack.juniper – directory containing all repository information related to the Juniper release. + +#. Install `direnv` using instructions on https://direnv.net/. Below you will find additional setup at the time of this writing so refer to latest of `direnv` site for additional configuration needed. + +#. Setup the following configuration to hook `direnv` for local directory environment overrides. There are two examples for BASH or ZSH (Mac OS X) shells. + + .. code:: sh + + ## ~/.bashrc for BASH shell + + ## Hook in `direnv` for local directory environment overrides. + ## https://direnv.net/docs/hook.html + eval "$(direnv hook bash)" + + # https://github.com/direnv/direnv/wiki/Python#bash + show_virtual_env() { + if [[ -n "$VIRTUAL_ENV" && -n "$DIRENV_DIR" ]]; then + echo "($(basename $VIRTUAL_ENV))" + fi + } + export -f show_virtual_env + PS1='$(show_virtual_env)'$PS1 + + # --------------------------------------------------- + + ## ~/.zshrc for ZSH shell for Mac OS X. + + ## Hook in `direnv` for local directory environment setup. + ## https://direnv.net/docs/hook.html + eval "$(direnv hook zsh)" + + # https://github.com/direnv/direnv/wiki/Python#zsh + setopt PROMPT_SUBST + + show_virtual_env() { + if [[ -n "$VIRTUAL_ENV" && -n "$DIRENV_DIR" ]]; then + echo "($(basename $VIRTUAL_ENV))" + fi + } + PS1='$(show_virtual_env)'$PS1 + +#. Setup `layout_python-venv` function to be used in local project directory `.envrc` file. + + .. code:: sh + + ## ~/.config/direnv/direnvrc + + # https://github.com/direnv/direnv/wiki/Python#venv-stdlib-module + + realpath() { + [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}" + } + layout_python-venv() { + local python=${1:-python3} + [[ $# -gt 0 ]] && shift + unset PYTHONHOME + if [[ -n $VIRTUAL_ENV ]]; then + VIRTUAL_ENV=$(realpath "${VIRTUAL_ENV}") + else + local python_version + python_version=$("$python" -c "import platform; print(platform.python_version())") + if [[ -z $python_version ]]; then + log_error "Could not detect Python version" + return 1 + fi + VIRTUAL_ENV=$PWD/.direnv/python-venv-$python_version + fi + export VIRTUAL_ENV + if [[ ! -d $VIRTUAL_ENV ]]; then + log_status "no venv found; creating $VIRTUAL_ENV" + "$python" -m venv "$VIRTUAL_ENV" + fi + + PATH="${VIRTUAL_ENV}/bin:${PATH}" + export PATH + } + +#. Example `.envrc` file used in project directory. Need to make sure that each release root has this unique file. + + .. code:: sh + + # Open edX named release project directory root. + ## /devstack.juniper/.envrc + + # https://discuss.openedx.org/t/docker-devstack-multiple-releases-one-machine/1902/10 + + # This is handled when OPENEDX_RELEASE is set. Leaving this in for manual override. + # export COMPOSE_PROJECT_NAME=devstack-juniper + + export DEVSTACK_WORKSPACE="$(pwd)" + export OPENEDX_RELEASE=juniper.master + export VIRTUAL_ENV="$(pwd)/devstack/venv" + + # https://github.com/direnv/direnv/wiki/Python#virtualenv + layout python-venv + How do I create relational database dumps? ------------------------------------------ We use relational database dumps to spend less time running relational database diff --git a/course-generator/create-courses.sh b/course-generator/create-courses.sh index eea96f6589..54e2bc3629 100755 --- a/course-generator/create-courses.sh +++ b/course-generator/create-courses.sh @@ -8,21 +8,21 @@ echo "Parsing options" container_error=false for arg in "$@"; do if [ $arg == "--studio" ]; then - if [ ! "$(docker-compose exec lms bash -c 'echo "Course will be created for studio"; exit $?')" ]; then + if [ ! "$(docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'echo "Course will be created for studio"; exit $?')" ]; then echo "Issue with studio container" container_error=true else studio=true fi elif [ $arg == "--ecommerce" ]; then - if [ ! "$(docker exec -t edx.devstack.ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then + if [ ! "$(docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then echo "Issue with ecommerce container" container_error=true else ecommerce=true fi elif [ $arg == "--marketing" ]; then - if [ ! "$(docker exec -t edx.devstack.marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then + if [ ! "$(docker-compose $DOCKER_COMPOSE_FILES exec marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then echo "Issue with marketing container. Course creation will proceed without marketing container." else marketing=true @@ -49,15 +49,15 @@ done < "${@: -1}" if $studio ; then echo "Creating courses on studio." - docker-compose exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'" + docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'" fi if $ecommerce ; then echo "Creating courses on ecommerce." - docker exec -t edx.devstack.ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" + docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" fi if $marketing ; then echo "Creating courses on marketing." - docker exec -t edx.devstack.marketing bash -c "drush generate_courses '$course_json'" + docker-compose $DOCKER_COMPOSE_FILES exec marketing bash -c "drush generate_courses '$course_json'" fi diff --git a/docker-compose-analytics-pipeline.yml b/docker-compose-analytics-pipeline.yml index aa84af84a6..01b9016acf 100644 --- a/docker-compose-analytics-pipeline.yml +++ b/docker-compose-analytics-pipeline.yml @@ -2,11 +2,15 @@ version: "2.1" services: namenode: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.namenode" image: edxops/analytics_pipeline_hadoop_namenode:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.namenode hostname: namenode environment: - CLUSTER_NAME=devstack + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.namenode ports: - 127.0.0.1:50070:50070 command: ["/run.sh"] @@ -14,13 +18,17 @@ services: - namenode_data:/hadoop/dfs/name datanode: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.datanode" image: edxops/analytics_pipeline_hadoop_datanode:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.datanode hostname: datanode environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" depends_on: - namenode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.datanode ports: - 127.0.0.1:50075:50075 command: ["/run.sh"] @@ -28,8 +36,8 @@ services: - datanode_data:/hadoop/dfs/data resourcemanager: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.resourcemanager" image: edxops/analytics_pipeline_hadoop_resourcemanager:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.resourcemanager hostname: resourcemanager environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" @@ -40,13 +48,17 @@ services: depends_on: - namenode - datanode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.resourcemanager ports: - 127.0.0.1:8088:8088 # resource manager web ui command: ["/run.sh"] nodemanager: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.nodemanager" image: edxops/analytics_pipeline_hadoop_nodemanager:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.nodemanager hostname: nodemanager environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" @@ -60,15 +72,23 @@ services: - resourcemanager - namenode - datanode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.nodemanager ports: - 127.0.0.1:8042:8042 # node manager web ui - 127.0.0.1:19888:19888 # node manager job history server ui command: ["/run.sh"] sparkmaster: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkmaster" image: edxops/analytics_pipeline_spark_master:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.sparkmaster hostname: sparkmaster + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.sparkmaster ports: - 127.0.0.1:8080:8080 - 127.0.0.1:7077:7077 # spark master port @@ -76,26 +96,34 @@ services: - 127.0.0.1:18080:18080 # spark history server sparkworker: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkworker" image: edxops/analytics_pipeline_spark_worker:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.sparkworker hostname: sparkworker depends_on: - sparkmaster environment: - SPARK_MASTER=spark://sparkmaster:7077 + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.sparkworker ports: - 127.0.0.1:8081:8081 # spark worker UI vertica: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.vertica" image: iamamr/vertica:9.1.0-0 hostname: vertica - container_name: edx.devstack.analytics_pipeline.vertica + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.vertica volumes: - vertica_data:/home/dbadmin/docker analyticspipeline: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline" image: edxops/analytics_pipeline:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline hostname: analyticspipeline volumes: - ${DEVSTACK_WORKSPACE}/edx-analytics-pipeline:/edx/app/analytics_pipeline/analytics_pipeline @@ -109,6 +137,10 @@ services: - sparkworker - elasticsearch - vertica + networks: + default: + aliases: + - edx.devstack.analytics_pipeline ports: - 127.0.0.1:4040:4040 # spark web UI environment: diff --git a/docker-compose-marketing-site.yml b/docker-compose-marketing-site.yml index 1051b4f872..bc48b89ce1 100644 --- a/docker-compose-marketing-site.yml +++ b/docker-compose-marketing-site.yml @@ -7,7 +7,7 @@ services: - MARKETING_SITE_ROOT="http://localhost:8080" marketing: - container_name: edx.devstack.marketing + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.marketing" depends_on: - mysql - memcached @@ -25,6 +25,10 @@ services: - DRUPAL_EXTRA_SETTINGS=${DRUPAL_EXTRA_SETTINGS:-/var/www/html/sites/default/docker.settings.php} # IP address of your machine to enable debugging (IP_ADDRESS set in .env file) - XDEBUG_CONFIG=remote_host=${XDEBUG_IP_ADDRESS:-127.0.0.1} - image: edxops/edx-mktg:latest + image: edxops/edx-mktg:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.marketing ports: - "8080:80" diff --git a/docker-compose-watchers-nfs.yml b/docker-compose-watchers-nfs.yml index 532a49060f..92fe18bdee 100644 --- a/docker-compose-watchers-nfs.yml +++ b/docker-compose-watchers-nfs.yml @@ -3,11 +3,15 @@ version: "2.1" services: lms_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.lms_watcher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.lms_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.lms_watcher volumes: - edx-nfs:/edx/app/edxapp/edx-platform - edxapp_lms_assets:/edx/var/edxapp/staticfiles/ @@ -17,11 +21,15 @@ services: studio_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.studio_watcher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.studio_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.studio_watcher volumes: - edxapp_studio_assets:/edx/var/edxapp/staticfiles/ - edx-nfs:/edx/app/edxapp/edx-platform diff --git a/docker-compose-watchers.yml b/docker-compose-watchers.yml index 7c7828f701..006d709d9d 100644 --- a/docker-compose-watchers.yml +++ b/docker-compose-watchers.yml @@ -3,11 +3,15 @@ version: "2.1" services: lms_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.lms_watcher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.lms_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.lms_watcher volumes: - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform:cached - edxapp_lms_assets:/edx/var/edxapp/staticfiles/ @@ -17,11 +21,15 @@ services: studio_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.studio_watcher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.studio_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.studio_watcher volumes: - edxapp_studio_assets:/edx/var/edxapp/staticfiles/ - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform:cached diff --git a/docker-compose-xqueue.yml b/docker-compose-xqueue.yml index bf4a73ac54..89e987c5ba 100644 --- a/docker-compose-xqueue.yml +++ b/docker-compose-xqueue.yml @@ -2,19 +2,27 @@ version: "2.1" services: xqueue: - container_name: edx.devstack.xqueue + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue" image: edxops/xqueue:${OPENEDX_RELEASE:-latest} command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py runserver 0.0.0.0:18040 ; sleep 2; done' volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file + networks: + default: + aliases: + - edx.devstack.xqueue ports: - 18040:18040 xqueue_consumer: - container_name: edx.devstack.xqueue_consumer + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue_consumer" image: edxops/xqueue:${OPENEDX_RELEASE:-latest} command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py run_consumer ; sleep 2; done' + networks: + default: + aliases: + - edx.devstack.xqueue_consumer volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file diff --git a/docker-compose.yml b/docker-compose.yml index 122ef80290..53bd811ba7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,10 +17,14 @@ services: # ================================================ chrome: - container_name: edx.devstack.chrome + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.chrome" hostname: chrome.devstack.edx image: edxops/chrome:${OPENEDX_RELEASE:-latest} shm_size: 2g + networks: + default: + aliases: + - edx.devstack.chrome ports: - "15900:5900" volumes: # for file uploads @@ -28,18 +32,26 @@ services: - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data devpi: - container_name: edx.devstack.devpi + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.devpi" hostname: devpi.devstack.edx image: edxops/devpi:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.devpi ports: - "3141:3141" volumes: - devpi_data:/data elasticsearch: - container_name: edx.devstack.elasticsearch + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.elasticsearch" hostname: elasticsearch.devstack.edx image: edxops/elasticsearch:devstack + networks: + default: + aliases: + - edx.devstack.elasticsearch # TODO: What to do about these forwarded ports? They'll conflict with ports forwarded by the Vagrant VM. # ports: # - "9200:9200" @@ -50,15 +62,23 @@ services: # This is meant to be used to test ES upgrades so that we do not have to upgrade all of our services to ES5 at once. elasticsearch-5: - container_name: edx.devstack.elasticsearch-5 + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.elasticsearch-5" hostname: elasticsearch-5.devstack.edx image: elasticsearch:5.6.16 + networks: + default: + aliases: + - edx.devstack.elasticsearch-5 firefox: - container_name: edx.devstack.firefox + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.firefox" hostname: firefox.devstack.edx image: edxops/firefox:${OPENEDX_RELEASE:-latest} shm_size: 2g + networks: + default: + aliases: + - edx.devstack.firefox ports: - "25900:5900" volumes: # for file uploads @@ -66,9 +86,13 @@ services: - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data memcached: - container_name: edx.devstack.memcached + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.memcached" hostname: memcached.devstack.edx image: memcached:1.5.10-alpine + networks: + default: + aliases: + - edx.devstack.memcached # ports: # - "11211:11211" @@ -77,9 +101,13 @@ services: # to conserve disk space, and disable the journal for a minor performance gain. # See https://docs.mongodb.com/v3.0/reference/program/mongod/#options for complete details. command: mongod --smallfiles --nojournal --storageEngine wiredTiger - container_name: edx.devstack.mongo + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mongo" hostname: mongo.devstack.edx image: mongo:${MONGO_VERSION:-3.6.17} + networks: + default: + aliases: + - edx.devstack.mongo # ports: # - "27017:27017" volumes: @@ -87,34 +115,46 @@ services: mysql: command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci - container_name: edx.devstack.mysql + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql" hostname: mysql.devstack.edx environment: MYSQL_ROOT_PASSWORD: "" MYSQL_ALLOW_EMPTY_PASSWORD: "yes" image: mysql:5.6 + networks: + default: + aliases: + - edx.devstack.mysql # ports: - # - "3506:3306" + # - "3506:3306" volumes: - mysql_data:/var/lib/mysql mysql57: command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci - container_name: edx.devstack.mysql57 + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql57" hostname: mysql57.devstack.edx environment: MYSQL_ROOT_PASSWORD: "" MYSQL_ALLOW_EMPTY_PASSWORD: "yes" image: mysql:5.7 + networks: + default: + aliases: + - edx.devstack.mysql57 # ports: # - "3506:3306" volumes: - mysql57_data:/var/lib/mysql redis: - container_name: edx.devstack.redis + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.redis" hostname: redis.devstack.edx image: redis:2.8 + networks: + default: + aliases: + - edx.devstack.redis command: redis-server --requirepass password # ================================================ @@ -123,7 +163,7 @@ services: credentials: command: bash -c 'source /edx/app/credentials/credentials_env && while true; do python /edx/app/credentials/credentials/manage.py runserver 0.0.0.0:18150; sleep 2; done' - container_name: edx.devstack.credentials + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.credentials" hostname: credentials.devstack.edx depends_on: - mysql @@ -139,12 +179,16 @@ services: ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/credentials:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.credentials ports: - "18150:18150" discovery: command: bash -c 'source /edx/app/discovery/discovery_env && while true; do python /edx/app/discovery/discovery/manage.py runserver 0.0.0.0:18381; sleep 2; done' - container_name: edx.devstack.discovery + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.discovery" hostname: discovery.devstack.edx depends_on: - mysql @@ -158,6 +202,10 @@ services: ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/discovery:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.discovery ports: - "18381:18381" volumes: @@ -165,7 +213,7 @@ services: ecommerce: command: bash -c 'source /edx/app/ecommerce/ecommerce_env && while true; do python /edx/app/ecommerce/ecommerce/manage.py runserver 0.0.0.0:18130; sleep 2; done' - container_name: edx.devstack.ecommerce + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.ecommerce" hostname: ecommerce.devstack.edx depends_on: - mysql @@ -179,18 +227,26 @@ services: DJANGO_WATCHMAN_TIMEOUT: 30 ENABLE_DJANGO_TOOLBAR: 1 image: edxops/ecommerce:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.ecommerce ports: - "18130:18130" edx_notes_api: command: bash -c 'source /edx/app/edx_notes_api/edx_notes_api_env && while true; do python /edx/app/edx_notes_api/edx_notes_api/manage.py runserver 0.0.0.0:18120 --settings notesserver.settings.devstack; sleep 2; done' - container_name: edx.devstack.edxnotesapi + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.edx_notes_api" hostname: edx_notes_api.devstack.edx depends_on: - devpi - elasticsearch - mysql image: edxops/notes:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.edxnotesapi ports: - "18120:18120" environment: @@ -206,19 +262,23 @@ services: forum: command: bash -c 'source /edx/app/forum/ruby_env && source /edx/app/forum/devstack_forum_env && cd /edx/app/forum/cs_comments_service && bundle install && while true; do ruby app.rb -o 0.0.0.0 ; sleep 2; done' - container_name: edx.devstack.forum + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.forum" hostname: forum.devstack.edx depends_on: - mongo - memcached - elasticsearch image: edxops/forum:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.forum ports: - "44567:4567" lms: command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack_docker; sleep 2; done' - container_name: edx.devstack.lms + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms" hostname: lms.devstack.edx depends_on: - devpi @@ -240,6 +300,10 @@ services: NO_PYTHON_UNINSTALL: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.lms ports: - "18000:18000" - "19876:19876" # JS test debugging @@ -250,7 +314,7 @@ services: registrar: command: bash -c 'source /edx/app/registrar/registrar_env && while true; do python /edx/app/registrar/registrar/manage.py runserver 0.0.0.0:18734; sleep 2; done' - container_name: edx.devstack.registrar + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.registrar" hostname: registrar.devstack.edx depends_on: - discovery @@ -278,6 +342,10 @@ services: CELERY_BROKER_PASSWORD: password DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/registrar:latest + networks: + default: + aliases: + - edx.devstack.registrar ports: - "18734:18734" volumes: @@ -285,7 +353,7 @@ services: registrar-worker: command: bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && celery -A registrar worker -l debug -c 2' - container_name: edx.devstack.registrar-worker + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.registrar-worker" hostname: registrar-worker.devstack.edx depends_on: - lms @@ -308,6 +376,10 @@ services: CELERY_BROKER_PASSWORD: password DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/registrar:latest + networks: + default: + aliases: + - edx.devstack.registrar-worker ports: - "18735:18735" volumes: @@ -315,7 +387,7 @@ services: studio: command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py cms runserver 0.0.0.0:18010 --settings devstack_docker; sleep 2; done' - container_name: edx.devstack.studio + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio" hostname: studio.devstack.edx depends_on: - devpi @@ -336,6 +408,10 @@ services: NO_PYTHON_UNINSTALL: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.studio ports: - "18010:18010" - "19877:19877" # JS test debugging @@ -353,33 +429,45 @@ services: # ========================================================================== frontend-app-publisher: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-publisher" extends: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/frontend-app-publisher' - container_name: edx.devstack.frontend-app-publisher + networks: + default: + aliases: + - edx.devstack.frontend-app-publisher ports: - "18400:18400" depends_on: - lms gradebook: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.gradebook" extends: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/gradebook' - container_name: edx.devstack.gradebook + networks: + default: + aliases: + - edx.devstack.gradebook ports: - "1994:1994" depends_on: - lms program-console: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.program-console" extends: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/program-console' - container_name: edx.devstack.program-console + networks: + default: + aliases: + - edx.devstack.program-console ports: - "1976:1976" depends_on: @@ -387,11 +475,15 @@ services: - registrar frontend-app-learning: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-learning" extends: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/frontend-app-learning' - container_name: edx.devstack.frontend-app-learning + networks: + default: + aliases: + - edx.devstack.frontend-app-learning ports: - "2000:2000" depends_on: diff --git a/enterprise/provision.sh b/enterprise/provision.sh index 559e02d671..cfb5676fd1 100755 --- a/enterprise/provision.sh +++ b/enterprise/provision.sh @@ -1,2 +1,2 @@ -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user enterprise_worker enterprise_worker@example.com --staff' -cat enterprise/worker_permissions.py | docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user enterprise_worker enterprise_worker@example.com --staff' +cat enterprise/worker_permissions.py | docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' diff --git a/gather-feature-toggle-state.sh b/gather-feature-toggle-state.sh index 2ed9946195..c78a73642b 100644 --- a/gather-feature-toggle-state.sh +++ b/gather-feature-toggle-state.sh @@ -12,20 +12,24 @@ if [ -e feature-toggle-data ]; then fi mkdir feature-toggle-data -docker exec -t edx.devstack.credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py dumpdata waffle --format=json > /edx/app/credentials/credentials/credentials_waffle.json' -docker cp edx.devstack.credentials:/edx/app/credentials/credentials/credentials_waffle.json feature-toggle-data -docker exec -t edx.devstack.credentials bash -c 'rm /edx/app/credentials/credentials/credentials_waffle.json' +credentials_id="$(make --silent dev.print-container.credentials)" +docker exec -t "$credentials_id" bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py dumpdata waffle --format=json > /edx/app/credentials/credentials/credentials_waffle.json' +docker cp "$credentials_id":/edx/app/credentials/credentials/credentials_waffle.json feature-toggle-data +docker exec -t "$credentials_id" bash -c 'rm /edx/app/credentials/credentials/credentials_waffle.json' -docker exec -t edx.devstack.discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py dumpdata waffle --format=json > /edx/app/discovery/discovery/discovery_waffle.json' -docker cp edx.devstack.discovery:/edx/app/discovery/discovery/discovery_waffle.json feature-toggle-data -docker exec -t edx.devstack.discovery bash -c '> /edx/app/discovery/discovery/discovery_waffle.json' +discovery_id="$(make --silent dev.print-container.discovery)" +docker exec -t "$discovery_id" bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py dumpdata waffle --format=json > /edx/app/discovery/discovery/discovery_waffle.json' +docker cp "$discovery_id".discovery:/edx/app/discovery/discovery/discovery_waffle.json feature-toggle-data +docker exec -t "$discovery_id" bash -c '> /edx/app/discovery/discovery/discovery_waffle.json' -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py dumpdata waffle --format=json > /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' -docker cp edx.devstack.ecommerce:/edx/app/ecommerce/ecommerce/ecommerce_waffle.json feature-toggle-data -docker exec -t edx.devstack.ecommerce bash -c 'rm /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' +ecommerce_id="$(make --silent dev.print-container.ecommerce)" +docker exec -t "$ecommerce_id" bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py dumpdata waffle --format=json > /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' +docker cp "$ecommerce_id".ecommerce:/edx/app/ecommerce/ecommerce/ecommerce_waffle.json feature-toggle-data +docker exec -t "$ecommerce_id" bash -c 'rm /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle --format=json > /edx/app/edxapp/edx-platform/lms_waffle.json' -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle_utils --format=json > /edx/app/edxapp/edx-platform/lms_waffle_utils.json' -docker cp edx.devstack.lms:/edx/app/edxapp/edx-platform/lms_waffle.json feature-toggle-data -docker cp edx.devstack.lms:/edx/app/edxapp/edx-platform/lms_waffle_utils.json feature-toggle-data -docker exec -t edx.devstack.lms bash -c 'rm /edx/app/edxapp/edx-platform/lms_waffle*.json' +lms_id="$(make --silent dev.print-container.lms)" +docker exec -t "$lms_id" bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle --format=json > /edx/app/edxapp/edx-platform/lms_waffle.json' +docker exec -t "$lms_id" bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle_utils --format=json > /edx/app/edxapp/edx-platform/lms_waffle_utils.json' +docker cp "$lms_id":/edx/app/edxapp/edx-platform/lms_waffle.json feature-toggle-data +docker cp "$lms_id":/edx/app/edxapp/edx-platform/lms_waffle_utils.json feature-toggle-data +docker exec -t "$lms_id" bash -c 'rm /edx/app/edxapp/edx-platform/lms_waffle*.json' diff --git a/load-db.sh b/load-db.sh index 24b70fada5..9b9aaa6c75 100755 --- a/load-db.sh +++ b/load-db.sh @@ -17,5 +17,6 @@ then fi echo "Loading the $1 database..." -docker exec -i edx.devstack.mysql mysql -uroot $1 < $1.sql +mysql_container=$(make -s dev.print-container.mysql) +docker exec "$mysql_container" bash -c "mysql -uroot $1" < $1.sql echo "Finished loading the $1 database!" diff --git a/marketing.mk b/marketing.mk index 16a8adb4a2..a8e1b4b59f 100644 --- a/marketing.mk +++ b/marketing.mk @@ -3,7 +3,7 @@ help-marketing: ## Display this help message @awk -F ':.*?## ' '/^[a-zA-Z]/ && NF==2 {printf "\033[36m %-25s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | grep marketing | sort marketing-shell: ## Run a shell on the marketing site container - docker exec -it edx.devstack.marketing env TERM=$(TERM) bash -c 'cd /edx/app/edx-mktg/edx-mktg; exec /bin/bash -sh' + docker-compose $(DOCKER_COMPOSE_FILES) exec marketing env TERM=$(TERM) bash -c 'cd /edx/app/edx-mktg/edx-mktg; exec /bin/bash -sh' stop-marketing: ## Stop all services (including the marketing site) with host volumes docker-compose -f docker-compose.yml -f docker-compose-host.yml -f docker-compose-themes.yml -f docker-compose-marketing-site.yml -f docker-compose-marketing-site-host.yml stop diff --git a/options.mk b/options.mk index d0f98018b7..b09035515b 100644 --- a/options.mk +++ b/options.mk @@ -8,6 +8,19 @@ # Defaults to parent. DEVSTACK_WORKSPACE ?= $(shell pwd)/.. +# Name of Docker Compose project. +# Volumes and network are namespaced based on this value, +# so changing it will give you a separate set of databases. +# See https://docs.docker.com/compose/reference/envvars/#compose_project_name +# If OPENEDX_RELAESE is defined, defaults to `devstack-${OPENEDX_RELEASE}`; +# otherwise, it defaults to `devstack`. +# Be sure to bring down services before changing the value of `COMPOSE_PROJECT_NAME`. +ifdef OPENEDX_RELEASE + COMPOSE_PROJECT_NAME ?= devstack-${OPENEDX_RELEASE} +else + COMPOSE_PROJECT_NAME ?= devstack +endif + # Name of Docker Compose project. # See https://docs.docker.com/compose/reference/envvars/#compose_project_name # Defaults to 'devstack'. @@ -34,7 +47,7 @@ FS_SYNC_STRATEGY ?= local-mounts # TODO: Re-evaluate this list and consider paring it down to a tighter core. # The current value was chosen such that it would not change the existing # Devstack behavior. -DEFAULT_SERVICES ?= lms+studio+ecommerce+discovery+credentials+forum+edx_notes_api+gradebook+frontend-app-publisher +DEFAULT_SERVICES ?= lms+studio+ecommerce+discovery+xqueue+credentials+forum+edx_notes_api+gradebook+frontend-app-publisher # List of all services with database migrations. # Services must provide a Makefile target named: $(service)-update-db diff --git a/programs/provision.sh b/programs/provision.sh index 967ae39774..3949babfe4 100755 --- a/programs/provision.sh +++ b/programs/provision.sh @@ -44,7 +44,7 @@ docker_exec() { /edx/app/$app/$repo/manage.py $cmd " - docker-compose exec -T "$service" bash -c "$CMDS" + docker-compose $DOCKER_COMPOSE_FILES exec -T "$service" bash -c "$CMDS" } provision_ida() { diff --git a/provision-analyticspipeline.sh b/provision-analyticspipeline.sh index 70e458e683..ffbe7e7176 100755 --- a/provision-analyticspipeline.sh +++ b/provision-analyticspipeline.sh @@ -22,24 +22,24 @@ docker-compose $DOCKER_COMPOSE_FILES up -d analyticspipeline # Analytics pipeline has dependency on lms but we only need its db schema & not full lms. So we'll just load their db # schemas as part of analytics pipeline provisioning. If there is a need of a fully fledged LMS, then provision lms # by following their documentation. -if [[ ! -z "`docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='edxapp'" 2>&1`" ]]; +if [[ ! -z "`docker-compose $DOCKER_COMPOSE_FILES exec mysql mysql -uroot -se "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='edxapp'" 2>&1`" ]]; then echo -e "${GREEN}LMS DB exists, skipping lms schema load.${NC}" else echo -e "${GREEN}LMS DB not found, provisioning lms schema.${NC}" - docker exec -i edx.devstack.mysql mysql -uroot mysql < provision.sql + docker-compose $DOCKER_COMPOSE_FILES exec mysql bash -c 'mysql -uroot mysql' < provision.sql ./load-db.sh edxapp docker-compose $DOCKER_COMPOSE_FILES up -d lms studio - docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' + docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' #Installing prereqs crashes the process docker-compose restart lms # Run edxapp migrations first since they are needed for the service users and OAuth clients - docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' + docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' fi echo -e "${GREEN}LMS database provisioned successfully...${NC}" echo -e "${GREEN}Creating databases and users...${NC}" -docker exec -i edx.devstack.mysql mysql -uroot mysql < provision-analytics-pipeline.sql +docker-compose $DOCKER_COMPOSE_FILES exec mysql bash -c 'mysql -uroot mysql' < provision-analytics-pipeline.sql # initialize hive metastore echo -e "${GREEN}Initializing HIVE metastore...${NC}" @@ -56,4 +56,4 @@ done sleep 10 # for datanode & other services to activate echo -e "${GREEN}Namenode is ready!${NC}" -docker exec -u hadoop -i edx.devstack.analytics_pipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' +docker-compose $DOCKER_COMPOSE_FILES exec -u hadoop analytics_pipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' diff --git a/provision-credentials.sh b/provision-credentials.sh index 32fccdaf61..1029ce339d 100755 --- a/provision-credentials.sh +++ b/provision-credentials.sh @@ -9,20 +9,20 @@ port=18150 docker-compose $DOCKER_COMPOSE_FILES up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" echo -e "${GREEN}Configuring site for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-discovery.sh b/provision-discovery.sh index 9b72e356df..d58f79351e 100755 --- a/provision-discovery.sh +++ b/provision-discovery.sh @@ -1,10 +1,10 @@ # Provisioning script for the discovery service ./provision-ida.sh discovery discovery 18381 -docker-compose exec -T discovery bash -c 'rm -rf /edx/var/discovery/*' -docker-compose exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 1 --site-domain localhost:18381 --code edx --name edX --courses-api-url "http://edx.devstack.lms:18000/api/courses/v1/" --ecommerce-api-url "http://edx.devstack.ecommerce:18130/api/v2/" --organizations-api-url "http://edx.devstack.lms:18000/api/organizations/v0/" --lms-url "http://edx.devstack.lms:18000/"' -docker-compose exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py refresh_course_metadata' -docker-compose exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py update_index --disable-change-limit' +docker-compose $DOCKER_COMPOSE_FILES exec -T discovery bash -c 'rm -rf /edx/var/discovery/*' +docker-compose $DOCKER_COMPOSE_FILES exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 1 --site-domain localhost:18381 --code edx --name edX --courses-api-url "http://edx.devstack.lms:18000/api/courses/v1/" --ecommerce-api-url "http://edx.devstack.ecommerce:18130/api/v2/" --organizations-api-url "http://edx.devstack.lms:18000/api/organizations/v0/" --lms-url "http://edx.devstack.lms:18000/"' +docker-compose $DOCKER_COMPOSE_FILES exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py refresh_course_metadata' +docker-compose $DOCKER_COMPOSE_FILES exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py update_index --disable-change-limit' # Add demo program ./programs/provision.sh discovery diff --git a/provision-e2e.sh b/provision-e2e.sh index b9e7b5508e..baee623cd0 100755 --- a/provision-e2e.sh +++ b/provision-e2e.sh @@ -10,14 +10,14 @@ elif [ ! -d "$DEVSTACK_WORKSPACE" ]; then fi # Copy the test course tarball into the studio container -docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz edx.devstack.studio:/tmp/ +docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz "$(make --silent dev.print-container.studio)":/tmp/ # Extract the test course tarball -docker-compose exec -T studio bash -c 'cd /tmp && tar xzf course.tar.gz' +docker-compose $DOCKER_COMPOSE_FILES exec -T studio bash -c 'cd /tmp && tar xzf course.tar.gz' # Import the course content -docker-compose exec -T studio bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker import /tmp course' +docker-compose $DOCKER_COMPOSE_FILES exec -T studio bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker import /tmp course' # Clean up the temp files -docker-compose exec -T studio bash -c 'rm /tmp/course.tar.gz' -docker-compose exec -T studio bash -c 'rm -r /tmp/course' +docker-compose $DOCKER_COMPOSE_FILES exec -T studio bash -c 'rm /tmp/course.tar.gz' +docker-compose $DOCKER_COMPOSE_FILES exec -T studio bash -c 'rm -r /tmp/course' diff --git a/provision-ecommerce.sh b/provision-ecommerce.sh index c1680fab02..bfefe42217 100755 --- a/provision-ecommerce.sh +++ b/provision-ecommerce.sh @@ -4,6 +4,6 @@ ./provision-ida.sh ecommerce ecommerce 18130 # Configure ecommerce -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/' -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' +docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/' +docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' +docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' diff --git a/provision-forum.sh b/provision-forum.sh index 6196ac0176..8e1f13b864 100755 --- a/provision-forum.sh +++ b/provision-forum.sh @@ -3,4 +3,4 @@ set -o pipefail set -x docker-compose $DOCKER_COMPOSE_FILES up -d forum -docker-compose exec -T forum bash -c 'source /edx/app/forum/ruby_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/' +docker-compose $DOCKER_COMPOSE_FILES exec -T forum bash -c 'source /edx/app/forum/ruby_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/' diff --git a/provision-ida-user.sh b/provision-ida-user.sh index 1a3b93c9ee..38e11e0a5a 100755 --- a/provision-ida-user.sh +++ b/provision-ida-user.sh @@ -7,8 +7,8 @@ client_port=$3 echo -e "${GREEN}Creating service user and OAuth2 applications for ${app_name}...${NC}" # Create the service user. -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" # Create the DOT applications - one for single sign-on and one for backend service IDA-to-IDA authentication. -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "http://localhost:$3/complete/edx-oauth2/" --client-id "$1-sso-key" --client-secret "$1-sso-secret" --scopes "user_id" $1-sso $1_worker' -- "$app_name" "$client_name" "$client_port" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "$1-backend-service-key" --client-secret "$1-backend-service-secret" $1-backend-service $1_worker' -- "$app_name" "$client_name" "$client_port" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "http://localhost:$3/complete/edx-oauth2/" --client-id "$1-sso-key" --client-secret "$1-sso-secret" --scopes "user_id" $1-sso $1_worker' -- "$app_name" "$client_name" "$client_port" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "$1-backend-service-key" --client-secret "$1-backend-service-secret" $1-backend-service $1_worker' -- "$app_name" "$client_name" "$client_port" diff --git a/provision-ida.sh b/provision-ida.sh index 62bbf4d0ad..e93282f28a 100755 --- a/provision-ida.sh +++ b/provision-ida.sh @@ -7,17 +7,17 @@ container_name=${4:-$1} # (Optional) The name of the container. If missing, wil docker-compose $DOCKER_COMPOSE_FILES up -d $app_name echo -e "${GREEN}Installing requirements for ${app_name}...${NC}" -docker exec -t edx.devstack.${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" echo -e "${GREEN}Running migrations for ${app_name}...${NC}" -docker exec -t edx.devstack.${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" echo -e "${GREEN}Creating super-user for ${app_name}...${NC}" -docker exec -t edx.devstack.${container_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec ${app_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" ./provision-ida-user.sh $app_name $client_name $client_port # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${app_name}...${NC}" -docker exec -t edx.devstack.${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" diff --git a/provision-lms.sh b/provision-lms.sh index 7ef1b8e252..6c69176741 100755 --- a/provision-lms.sh +++ b/provision-lms.sh @@ -13,33 +13,33 @@ for app in "${apps[@]}"; do docker-compose $DOCKER_COMPOSE_FILES up -d $app done -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' #Installing prereqs crashes the process docker-compose restart lms # Run edxapp migrations first since they are needed for the service users and OAuth clients -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' # Create a superuser for edxapp -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user edx edx@example.com --superuser --staff' -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username=\"edx\"); user.set_password(\"edx\"); user.save()" | python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user edx edx@example.com --superuser --staff' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username=\"edx\"); user.set_password(\"edx\"); user.save()" | python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' # Create an enterprise service user for edxapp and give them appropriate permissions ./enterprise/provision.sh # Enable the LMS-E-Commerce integration -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce' # Create demo course and users -docker-compose exec -T lms bash -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"' # Fix missing vendor file by clearing the cache -docker-compose exec -T lms bash -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1' # Create static assets for both LMS and Studio for app in "${apps[@]}"; do - docker-compose exec -T $app bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker' + docker-compose $DOCKER_COMPOSE_FILES exec -T $app bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker' done # Provision a retirement service account user diff --git a/provision-notes.sh b/provision-notes.sh index a840408741..4341580dfd 100755 --- a/provision-notes.sh +++ b/provision-notes.sh @@ -5,4 +5,4 @@ # This will build the elasticsearch index for notes. echo -e "${GREEN}Creating indexes for edx_notes_api...${NC}" -docker exec -t edx.devstack.edxnotesapi bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py rebuild_index --noinput' -- edx_notes_api +docker-compose $DOCKER_COMPOSE_FILES exec edx_notes_api bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py rebuild_index --noinput' -- edx_notes_api diff --git a/provision-registrar.sh b/provision-registrar.sh index 3fd78f683c..e2b30e1ebf 100755 --- a/provision-registrar.sh +++ b/provision-registrar.sh @@ -6,17 +6,17 @@ port=18734 docker-compose $DOCKER_COMPOSE_FILES up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make requirements' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make migrate' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make createsuperuser' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make createsuperuser' -- "$name" ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c ' if ! source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make static 2>registrar_make_static.err; then echo "------- Last 100 lines of stderr"; tail regsitrar_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c ' if ! source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make static 2>registrar_make_static.err; then echo "------- Last 100 lines of stderr"; tail regsitrar_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-retirement-user.sh b/provision-retirement-user.sh index b3a5dac2e3..9a9f209c6c 100755 --- a/provision-retirement-user.sh +++ b/provision-retirement-user.sh @@ -4,5 +4,5 @@ app_name=$1 user_name=$2 echo -e "${GREEN}Creating retirement service user ${user_name} and DOT Application ${app_name}...${NC}" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" diff --git a/provision.sh b/provision.sh index 60e6e04bfe..220be49111 100755 --- a/provision.sh +++ b/provision.sh @@ -122,7 +122,7 @@ fi # Ensure the MySQL server is online and usable echo "${GREEN}Waiting for MySQL.${NC}" -until docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')" &> /dev/null +until docker-compose $DOCKER_COMPOSE_FILES exec -T mysql bash -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 @@ -136,20 +136,20 @@ echo -e "${GREEN}MySQL ready.${NC}" # Ensure that the MySQL databases and users are created for all IDAs. # (A no-op for databases and users that already exist). echo -e "${GREEN}Ensuring MySQL databases and users exist...${NC}" -docker exec -i edx.devstack.mysql mysql -uroot mysql < provision.sql +docker-compose $DOCKER_COMPOSE_FILES exec -T mysql bash -c "mysql -uroot mysql" < provision.sql # If necessary, ensure the MongoDB server is online and usable # and create its users. if needs_mongo "$to_provision"; then echo -e "${GREEN}Waiting for MongoDB...${NC}" - until docker exec -i edx.devstack.mongo mongo --eval "printjson(db.serverStatus())" &> /dev/null + until docker-compose $DOCKER_COMPOSE_FILES exec -T mongo bash -c 'mongo --eval "printjson(db.serverStatus())" &> /dev/null' do printf "." sleep 1 done echo -e "${GREEN}MongoDB ready.${NC}" echo -e "${GREEN}Creating MongoDB users...${NC}" - docker exec -i edx.devstack.mongo mongo < mongo-provision.js + docker-compose $DOCKER_COMPOSE_FILES exec -T mongo bash -c 'mongo' < mongo-provision.js else echo -e "${GREEN}MongoDB preparation not required; skipping.${NC}" fi diff --git a/update-dbs-init-sql-scripts.sh b/update-dbs-init-sql-scripts.sh index 53d9830ee0..591144434b 100755 --- a/update-dbs-init-sql-scripts.sh +++ b/update-dbs-init-sql-scripts.sh @@ -24,8 +24,7 @@ make dev.clone.ssh make dev.provision.services.lms+ecommerce # dump schema and data from mysql databases in the mysql docker container and copy them to current directory in docker host -MYSQL_DOCKER_COMPOSE_SERVICE="mysql" -MYSQL_DOCKER_CONTAINER="edx.devstack.${MYSQL_DOCKER_COMPOSE_SERVICE}" +MYSQL_DOCKER_CONTAINER="$(make --silent dev.print-container.mysql)" for DB_NAME in "${DBS[@]}"; do DB_CREATION_SQL_SCRIPT="${DB_NAME}.sql" if [[ " ${EDXAPP_DBS[@]} " =~ " ${DB_NAME} " ]]; then @@ -33,6 +32,6 @@ for DB_NAME in "${DBS[@]}"; do else MYSQL_DB_USER=${ECOMMERCE_MYSQL_DB_USER} fi - docker exec ${MYSQL_DOCKER_CONTAINER} /bin/bash -c "mysqldump -u ${MYSQL_DB_USER} -p${MYSQL_DB_PASSWORD} --add-drop-database --skip-add-drop-table --databases ${DB_NAME} > ${DB_CREATION_SQL_SCRIPT}" + docker-compose $DOCKER_COMPOSE_FILES exec ${MYSQL_DOCKER_CONTAINER} /bin/bash -c "mysqldump -u ${MYSQL_DB_USER} -p${MYSQL_DB_PASSWORD} --add-drop-database --skip-add-drop-table --databases ${DB_NAME} > ${DB_CREATION_SQL_SCRIPT}" docker cp ${MYSQL_DOCKER_CONTAINER}:/${DB_CREATION_SQL_SCRIPT} . done diff --git a/upgrade_mongo_3_6.sh b/upgrade_mongo_3_6.sh index 1d0ce7c9c2..28ec53480e 100755 --- a/upgrade_mongo_3_6.sh +++ b/upgrade_mongo_3_6.sh @@ -11,11 +11,12 @@ export MONGO_VERSION=3.4.24 echo -e "${GREEN}Sarting Mongo ${MONGO_VERSION}${NC}" docker-compose up -d mongo +mongo_container="$(make -s dev.print-container.mongo)" echo -e "${GREEN}Waiting for MongoDB...${NC}" -until docker exec -i edx.devstack.mongo mongo --eval "printjson(db.serverStatus())" &> /dev/null +until docker exec "$mongo_container" bash -c 'mongo --eval "printjson(db.serverStatus())" &> /dev/null' do - if docker-compose logs mongo | grep -q "BadValue: Invalid value for version, found 3.6, expected '3.4' or '3.2'"; then + if docker logs "$mongo_container" | grep -q "BadValue: Invalid value for version, found 3.6, expected '3.4' or '3.2'"; then echo -e "${YELLOW}Already upgraded to Mongo 3.6, exiting${NC}" exit fi @@ -24,15 +25,15 @@ do done echo -e "${GREEN}MongoDB ready.${NC}" -MONGO_VERSION_LIVE=$(docker-compose exec mongo mongo --quiet --eval "printjson(db.version())") -MONGO_VERSION_COMPAT=$(docker-compose exec mongo mongo --quiet \ +MONGO_VERSION_LIVE=$(docker-compose $DOCKER_COMPOSE_FILES exec -it "$mongo_container" mongo --quiet --eval "printjson(db.version())") +MONGO_VERSION_COMPAT=$(docker-compose $DOCKER_COMPOSE_FILES exec -it "$mongo_container" mongo --quiet \ --eval "printjson(db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )['featureCompatibilityVersion'])") echo -e "${GREEN}Mongo Server version: ${MONGO_VERSION_LIVE}${NC}" echo -e "${GREEN}Mongo FeatureCompatibilityVersion version: ${MONGO_VERSION_COMPAT}${NC}" if echo "${MONGO_VERSION_COMPAT}" | grep -q "3\.2" ; then echo -e "${GREEN}Upgrading FeatureCompatibilityVersion to 3.4${NC}" - docker-compose exec mongo mongo --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"3.4\" } )" + docker exec -it "$mongo_container" mongo --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"3.4\" } )" else echo -e "${GREEN}FeatureCompatibilityVersion already set to 3.4${NC}" fi @@ -44,23 +45,25 @@ echo echo -e "${GREEN}Restarting Mongo on version ${MONGO_VERSION}${NC}" docker-compose up -d mongo +mongo_container="$(make -s dev.print-container.mongo)" + echo -e "${GREEN}Waiting for MongoDB...${NC}" -until docker exec -i edx.devstack.mongo mongo --eval "printjson(db.serverStatus())" &> /dev/null +until docker-compose $DOCKER_COMPOSE_FILES exec mongo bash -c 'mongo --eval "printjson(db.serverStatus())" &> /dev/null' do printf "." sleep 1 done echo -e "${GREEN}MongoDB ready.${NC}" -MONGO_VERSION_LIVE=$(docker-compose exec mongo mongo --quiet --eval "printjson(db.version())") -MONGO_VERSION_COMPAT=$(docker-compose exec mongo mongo --quiet \ +MONGO_VERSION_LIVE=$(docker-compose $DOCKER_COMPOSE_FILES exec -it "$mongo_container" mongo --quiet --eval "printjson(db.version())") +MONGO_VERSION_COMPAT=$(docker-compose $DOCKER_COMPOSE_FILES exec -it "$mongo_container" mongo --quiet \ --eval "printjson(db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )['featureCompatibilityVersion'])") echo -e "${GREEN}Mongo Server version: ${MONGO_VERSION_LIVE}${NC}" echo -e "${GREEN}Mongo FeatureCompatibilityVersion version: ${MONGO_VERSION_COMPAT}${NC}" if echo "${MONGO_VERSION_COMPAT}" | grep -q "3\.4" ; then echo -e "${GREEN}Upgrading FeatureCompatibilityVersion to 3.6${NC}" - docker-compose exec mongo mongo --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"3.6\" } )" + docker exec -it "$mongo_container" mongo --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"3.6\" } )" else echo -e "${GREEN}FeatureCompatibilityVersion already set to 3.6${NC}" fi