diff --git a/Makefile b/Makefile index 1134c14c5e..a4dfd3770f 100644 --- a/Makefile +++ b/Makefile @@ -9,11 +9,12 @@ .PHONY: analytics-pipeline-devstack-test analytics-pipeline-shell \ analyticspipeline-shell backup build-courses check-memory \ create-test-course credentials-shell destroy dev.cache-programs \ - dev.check dev.checkout dev.clone dev.clone.ssh dev.nfs.setup \ - devpi-password dev.provision dev.provision.analytics_pipeline \ - dev.provision.services dev.provision.xqueue dev.pull dev.repo.reset \ - dev.reset dev.status dev.sync.daemon.start dev.sync.provision \ - dev.sync.requirements dev.sync.up dev.up dev.up.all \ + dev.check dev.checkout dev.clone dev.clone.ssh dev.down dev.kill \ + dev.nfs.setup devpi-password dev.provision \ + dev.provision.analytics_pipeline dev.provision.services \ + dev.provision.xqueue dev.ps dev.pull dev.repo.reset dev.reset \ + dev.restart dev.rm-stopped dev.status dev.stop dev.sync.daemon.start \ + dev.sync.provision dev.sync.requirements dev.sync.up dev.up dev.up.all \ dev.up.analytics_pipeline dev.up.watchers dev.up.with-programs \ discovery-shell down e2e-shell e2e-tests ecommerce-shell \ feature-toggle-state forum-restart-devserver healthchecks help \ @@ -37,6 +38,8 @@ include options.mk -include local.mk # Prefix with hyphen to tolerate absence of file. # Docker Compose YAML files to define services and their volumes. +# This environment variable tells `docker-compose` which files to load definitions +# of services, volumes, and networks from. # Depending on the value of FS_SYNC_STRATEGY, we use a slightly different set of # files, enabling use of different strategies to synchronize files between the host and # the containers. @@ -44,34 +47,34 @@ include options.mk # For example, the LMS/Studio asset watchers are only available for local-mounts and nfs, # and XQueue and the Analytics Pipeline are only available for local-mounts. +# Compose files are separated by a colon. +COMPOSE_PATH_SEPARATOR := : + ifeq ($(FS_SYNC_STRATEGY),local-mounts) -DOCKER_COMPOSE_FILES := \ --f docker-compose-host.yml \ --f docker-compose-themes.yml \ --f docker-compose-watchers.yml \ --f docker-compose-xqueue.yml \ --f docker-compose-analytics-pipeline.yml \ --f docker-compose-marketing-site.yml +COMPOSE_FILE := docker-compose-host.yml +COMPOSE_FILE := $(COMPOSE_FILE):docker-compose-themes.yml +COMPOSE_FILE := $(COMPOSE_FILE):docker-compose-watchers.yml +COMPOSE_FILE := $(COMPOSE_FILE):docker-compose-xqueue.yml +COMPOSE_FILE := $(COMPOSE_FILE):docker-compose-analytics-pipeline.yml +COMPOSE_FILE := $(COMPOSE_FILE):docker-compose-marketing-site.yml endif ifeq ($(FS_SYNC_STRATEGY),nfs) -DOCKER_COMPOSE_FILES := \ --f docker-compose-host-nfs.yml \ --f docker-compose-themes-nfs.yml \ --f docker-compose-watchers-nfs.yml +COMPOSE_FILE := docker-compose-host-nfs.yml +COMPOSE_FILE := $(COMPOSE_FILE):docker-compose-themes-nfs.yml +COMPOSE_FILE := $(COMPOSE_FILE):docker-compose-watchers-nfs.yml endif ifeq ($(FS_SYNC_STRATEGY),docker-sync) -DOCKER_COMPOSE_FILES := \ --f docker-compose-sync.yml +COMPOSE_FILE := docker-compose-sync.yml endif -ifndef DOCKER_COMPOSE_FILES +ifndef COMPOSE_FILE $(error FS_SYNC_STRATEGY is set to $(FS_SYNC_STRATEGY). Must be one of: local-mounts, nfs, docker-sync) endif # All three filesystem synchronization strategy require the main docker-compose.yml file. -DOCKER_COMPOSE_FILES := -f docker-compose.yml $(DOCKER_COMPOSE_FILES) +COMPOSE_FILE := docker-compose.yml:$(COMPOSE_FILE) OS := $(shell uname) @@ -110,6 +113,12 @@ upgrade: ## Upgrade requirements with pip-tools pip-compile --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in pip-compile --upgrade -o requirements/base.txt requirements/base.in +dev.print-container.%: ## Get the ID of the running container for a given service. Run with ``make --silent`` for just ID. + @echo $$(docker-compose ps --quiet $*) + +dev.ps: ## View list of created services and their statuses. + docker-compose ps + dev.checkout: ## Check out "openedx-release/$OPENEDX_RELEASE" in each repo if set, "master" otherwise ./repo.sh checkout @@ -147,21 +156,21 @@ dev.repo.reset: ## Attempts to reset the local repo checkouts to the master work dev.pull: dev.pull.$(DEFAULT_SERVICES) ## Pull Docker images required by default services. dev.pull.without-deps.%: ## Pull latest Docker images for services (separated by plus-signs). - docker-compose $(DOCKER_COMPOSE_FILES) pull $$(echo $* | tr + " ") + docker-compose pull $$(echo $* | tr + " ") dev.pull.%: ## Pull latest Docker images for services (separated by plus-signs) and all their dependencies. - docker-compose $(DOCKER_COMPOSE_FILES) pull --include-deps $$(echo $* | tr + " ") + docker-compose pull --include-deps $$(echo $* | tr + " ") dev.up: dev.up.$(DEFAULT_SERVICES) check-memory ## Bring up default services. dev.up.%: | check-memory ## Bring up specific services (separated by plus-signs) and their dependencies with host volumes. - docker-compose $(DOCKER_COMPOSE_FILES) up -d $$(echo $* | tr + " ") + docker-compose up -d $$(echo $* | tr + " ") ifeq ($(ALWAYS_CACHE_PROGRAMS),true) make dev.cache-programs endif dev.up.without-deps.%: ## Bring up specific services (separated by plus-signs) without dependencies. - docker-compose $(DOCKER_COMPOSE_FILES) up --d --no-deps $$(echo $* | tr + " ") + docker-compose up --d --no-deps $$(echo $* | tr + " ") dev.up.with-programs: dev.up dev.cache-programs ## Bring up a all services and cache programs in LMS. @@ -200,30 +209,54 @@ dev.check.%: # Run checks for a given service or set of services (separated by provision: | dev.provision ## This command will be deprecated in a future release, use dev.provision echo "\033[0;31mThis command will be deprecated in a future release, use dev.provision\033[0m" -stop: ## Stop all services +dev.stop: ## Stop all services. + (test -d .docker-sync && docker-sync stop) || true ## Ignore failure here + docker-compose stop + +dev.stop.%: ## Stop specific services, separated by plus-signs. + docker-compose stop $$(echo $* | tr + " ") + +stop: dev.stop.$(DEFAULT_SERVICES) + +stop.watchers: dev.stop.lms_watcher+studio_watcher + +stop.all: dev.stop + +stop.xqueue: dev.stop.xqueue+xqueue_consumer + +dev.restart: ## Restart all services. + docker-compose restart $$(echo $* | tr + " ") + +dev.restart.%: ## Restart specific services, separated by plus-signs. + docker-compose restart $$(echo $* | tr + " ") + +dev.kill: ## Kill all services. (test -d .docker-sync && docker-sync stop) || true ## Ignore failure here docker-compose stop -stop.watchers: ## Stop asset watchers - docker-compose $(DOCKER_COMPOSE_FILES) stop lms_watcher studio_watcher +dev.kill.%: ## Kill specific services, separated by plus-signs. + docker-compose kill $$(echo $* | tr + " ") -stop.all: | stop.analytics_pipeline stop stop.watchers ## Stop all containers, including asset watchers +dev.rm-stopped: ## Remove stopped containers. Does not affect running containers. + docker-compose rm --force -stop.xqueue: ## Stop the XQueue and XQueue-Consumer containers - docker-compose $(DOCKER_COMPOSE_FILES) stop xqueue xqueue_consumer +dev.down.%: ## Stop and remove specific services, separated by plus-signs. + docker-compose rm --force --stop $$(echo $* | tr + " ") -down: ## Remove all service containers and networks +dev.down: ## Stop and remove all service containers and networks (test -d .docker-sync && docker-sync clean) || true ## Ignore failure here - docker-compose $(DOCKER_COMPOSE_FILES) down + docker-compose down + +down: dev.down destroy: ## Remove all devstack-related containers, networks, and volumes $(WINPTY) bash ./destroy.sh logs: ## View logs from containers running in detached mode - docker-compose $(DOCKER_COMPOSE_FILES) logs -f + docker-compose logs -f %-logs: ## View the logs of the specified service container - docker-compose $(DOCKER_COMPOSE_FILES) logs -f --tail=500 $* + docker-compose logs -f --tail=500 $* RED="\033[0;31m" YELLOW="\033[0;33m" @@ -266,93 +299,93 @@ validate: ## Validate the devstack configuration docker-compose config backup: dev.up.mysql+mongo+elasticsearch ## Write all data volumes to the host. - docker run --rm --volumes-from edx.devstack.mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql - docker run --rm --volumes-from edx.devstack.mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db - docker run --rm --volumes-from edx.devstack.elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data + docker run --rm --volumes-from $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql + docker run --rm --volumes-from $$(make -s dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db + docker run --rm --volumes-from $$(make -s dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data restore: dev.up.mysql+mongo+elasticsearch ## Restore all data volumes from the host. WARNING: THIS WILL OVERWRITE ALL EXISTING DATA! - docker run --rm --volumes-from edx.devstack.mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz - docker run --rm --volumes-from edx.devstack.mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz - docker run --rm --volumes-from edx.devstack.elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz + docker run --rm --volumes-from $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz + docker run --rm --volumes-from $$(make -s dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz + docker run --rm --volumes-from $$(make -s dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz # TODO: Print out help for this target. Even better if we can iterate over the # services in docker-compose.yml, and print the actual service names. %-shell: ## Run a shell on the specified service container - docker exec -it edx.devstack.$* /bin/bash + docker-compose exec $* /bin/bash analyticspipeline-shell: ## Run a shell on the analytics pipeline container - docker exec -it edx.devstack.analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open + docker-compose exec analyticspipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open credentials-shell: ## Run a shell on the credentials container - docker exec -it edx.devstack.credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' + docker-compose exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' discovery-shell: ## Run a shell on the discovery container - docker exec -it edx.devstack.discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open + docker-compose exec discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open ecommerce-shell: ## Run a shell on the ecommerce container - docker exec -it edx.devstack.ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open + docker-compose exec ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open e2e-shell: ## Start the end-to-end tests container with a shell - docker run -it --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash + docker run -it --network=${COMPOSE_PROJECT_NAME:-devstack}_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash registrar-shell: ## Run a shell on the registrar site container - docker exec -it edx.devstack.registrar env TERM=$(TERM) /edx/app/registrar/devstack.sh open + docker-compose exec registrar env TERM=$(TERM) /edx/app/registrar/devstack.sh open %-update-db: ## Run migrations for the specified service container - docker exec -t edx.devstack.$* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' + docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' studio-update-db: ## Run migrations for the Studio container - docker exec -t edx.devstack.studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' + docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' lms-update-db: ## Run migrations LMS container - docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' + docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' update-db: | $(DB_MIGRATION_TARGETS) ## Run the migrations for DEFAULT_SERVICES forum-restart-devserver: ## Kill the forum's Sinatra development server. The watcher process will restart it. - docker exec -t edx.devstack.forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose exec forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")' lms-shell: ## Run a shell on the LMS container - docker exec -it edx.devstack.lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose exec lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open lms-watcher-shell: ## Run a shell on the LMS watcher container - docker exec -it edx.devstack.lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose exec lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open %-attach: ## Attach to the specified service container process to use the debugger & see logs. - docker attach edx.devstack.$* + docker attach "$$(make --silent dev.print-container.$*)" lms-restart: lms-restart-devserver studio-shell: ## Run a shell on the Studio container - docker exec -it edx.devstack.studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose exec studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-watcher-shell: ## Run a shell on the studio watcher container - docker exec -it edx.devstack.studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose exec studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-restart: studio-restart-devserver xqueue-shell: ## Run a shell on the XQueue container - docker exec -it edx.devstack.xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open + docker-compose exec xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue-restart: xqueue-restart-devserver %-restart-devserver: ## Kill a service's Django development server. The watcher process should restart it. - docker exec -t edx.devstack.$* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose exec $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' xqueue_consumer-shell: ## Run a shell on the XQueue consumer container - docker exec -it edx.devstack.xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open + docker-compose exec xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue_consumer-restart: ## Kill the XQueue development server. The watcher process will restart it. - docker exec -t edx.devstack.xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose exec xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")' %-static: ## Rebuild static assets for the specified service container - docker exec -t edx.devstack.$* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' + docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' lms-static: ## Rebuild static assets for the LMS container - docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets lms' + docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets lms' studio-static: ## Rebuild static assets for the Studio container - docker exec -t edx.devstack.studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets studio' + docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets studio' static: | credentials-static discovery-static ecommerce-static lms-static studio-static ## Rebuild static assets for all service containers @@ -361,16 +394,16 @@ healthchecks: dev.check.$(DEFAULT_SERVICES) healthchecks.%: dev.check.% e2e-tests: ## Run the end-to-end tests against the service containers - docker run -t --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash -c 'paver e2e_test --exclude="whitelabel\|enterprise"' + docker run -t --network=${COMPOSE_PROJECT_NAME:-devstack}_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash -c 'paver e2e_test --exclude="whitelabel\|enterprise"' validate-lms-volume: ## Validate that changes to the local workspace are reflected in the LMS container touch $(DEVSTACK_WORKSPACE)/edx-platform/testfile - docker exec edx.devstack.lms ls /edx/app/edxapp/edx-platform/testfile + docker-compose exec -T lms ls /edx/app/edxapp/edx-platform/testfile rm $(DEVSTACK_WORKSPACE)/edx-platform/testfile vnc-passwords: ## Get the VNC passwords for the Chrome and Firefox Selenium containers - @docker logs edx.devstack.chrome 2>&1 | grep "VNC password" | tail -1 - @docker logs edx.devstack.firefox 2>&1 | grep "VNC password" | tail -1 + @docker-compose logs chrome 2>&1 | grep "VNC password" | tail -1 + @docker-compose logs firefox 2>&1 | grep "VNC password" | tail -1 devpi-password: ## Get the root devpi password for the devpi container docker-compose exec devpi bash -c "cat /data/server/.serverpassword" @@ -393,15 +426,12 @@ dev.up.analytics_pipeline: dev.up.analyticspipeline ## Bring up analytics pipeli pull.analytics_pipeline: dev.pull.analyticspipeline ## Update analytics pipeline docker images analytics-pipeline-devstack-test: ## Run analytics pipeline tests in travis build - docker exec -u hadoop -i edx.devstack.analytics_pipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity' + docker-compose exec -u hadoop -T analyticspipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity' -stop.analytics_pipeline: ## Stop all Analytics pipeline services. - docker-compose $(DOCKER_COMPOSE_FILES) stop \ - namenode datanode resourcemanager nodemanager sparkmaster \ - sparkworker vertica analyticspipeline +stop.analytics_pipeline: dev.stop.namenode+datanode+resourcemanager+nodemanager+sparkmaster+sparkworker+vertica+analyticspipeline ## Stop all Analytics pipeline services. hadoop-application-logs-%: ## View hadoop logs by application Id - docker exec -it edx.devstack.analytics_pipeline.nodemanager yarn logs -applicationId $* + docker-compose exec nodemanager yarn logs -applicationId $* # Provisions studio, ecommerce, and marketing with course(s) in test-course.json # Modify test-course.json before running this make target to generate a custom course diff --git a/Makefile.edx b/Makefile.edx index 81343ba2de..f089acd155 100644 --- a/Makefile.edx +++ b/Makefile.edx @@ -23,8 +23,8 @@ dev.provision.whitelabel: # The containers must be started with the 'dev.up.e2e_wl_tests' target. # AND the test must be setup using the 'dev.provision.whitelabel' target. whitelabel-tests: - docker run -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e - docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh devstack.whitelabel:/tmp/run_whitelabel_tests.sh + docker run -d --name=devstack.whitelabel --network=${COMPOSE_PROJECT_NAME:-devstack}_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e + docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh $(make --silent dev.print-container.devstack.whitelabel)":/tmp/run_whitelabel_tests.sh docker exec -t devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh whitelabel-cleanup: diff --git a/README.rst b/README.rst index d3eb1fba0f..913b317c0f 100644 --- a/README.rst +++ b/README.rst @@ -405,11 +405,11 @@ already being correcty provisioned. So, when in doubt, it may still be best to run the full ``make dev.provision``. Sometimes you may need to restart a particular application server. To do so, -simply use the ``docker-compose restart`` command: +simply use the ``make dev.restart.%`` command: .. code:: sh - docker-compose restart + make dev.restart. In all the above commands, ```` should be replaced with one of the following: @@ -681,7 +681,7 @@ vary depending on the database. For all of the options, see ``provision.sql``. - Password: ``password`` If you have trouble connecting, ensure the port was mapped successfully by -running ``docker-compose ps`` and looking for a line like this: +running ``make dev.ps`` and looking for a line like this: ``edx.devstack.mysql docker-entrypoint.sh mysql ... Up 0.0.0.0:3506→3306/tcp``. Switching branches @@ -774,20 +774,20 @@ Set a PDB breakpoint anywhere in the code using: and your attached session will offer an interactive PDB prompt when the breakpoint is hit. -To detach from the container, you'll need to stop the container with: +You may be able to detach from the container with the ``Ctrl-P, Ctrl-Q`` key sequence. +If that doesn't work, you will have either close your terminal window, +stop the container with: .. code:: sh - make stop + make dev.stop. -or a manual Docker command to bring down the container: +or kill the container with: .. code:: sh - docker kill $(docker ps -a -q --filter="name=edx.devstack.") + make dev.kill. -Alternatively, some terminals allow detachment from a running container with the -``Ctrl-P, Ctrl-Q`` key sequence. Running LMS and Studio Tests ---------------------------- @@ -872,7 +872,7 @@ Check the logs If a container stops unexpectedly, you can look at its logs for clues:: - docker-compose logs lms + make -logs Update the code and images ~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/course-generator/create-courses.sh b/course-generator/create-courses.sh index eea96f6589..b7ee059a9e 100755 --- a/course-generator/create-courses.sh +++ b/course-generator/create-courses.sh @@ -15,14 +15,14 @@ for arg in "$@"; do studio=true fi elif [ $arg == "--ecommerce" ]; then - if [ ! "$(docker exec -t edx.devstack.ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then + if [ ! "$(docker-compose exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then echo "Issue with ecommerce container" container_error=true else ecommerce=true fi elif [ $arg == "--marketing" ]; then - if [ ! "$(docker exec -t edx.devstack.marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then + if [ ! "$(docker-compose exec marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then echo "Issue with marketing container. Course creation will proceed without marketing container." else marketing=true @@ -54,10 +54,10 @@ fi if $ecommerce ; then echo "Creating courses on ecommerce." - docker exec -t edx.devstack.ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" + docker-compose exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" fi if $marketing ; then echo "Creating courses on marketing." - docker exec -t edx.devstack.marketing bash -c "drush generate_courses '$course_json'" + docker-compose exec marketing bash -c "drush generate_courses '$course_json'" fi diff --git a/destroy.sh b/destroy.sh index f00ea0f6cc..da47f02cb6 100755 --- a/destroy.sh +++ b/destroy.sh @@ -5,5 +5,5 @@ set -e read -p "This will delete all data in your devstack. Would you like to proceed? [y/n] " -r if [[ $REPLY =~ ^[Yy]$ ]] then - docker-compose -f docker-compose.yml -f docker-compose-watchers.yml -f docker-compose-host.yml -f docker-compose-themes.yml -f docker-compose-analytics-pipeline.yml down -v + docker-compose down -v fi diff --git a/docker-compose-analytics-pipeline.yml b/docker-compose-analytics-pipeline.yml index aa84af84a6..5cc50a0438 100644 --- a/docker-compose-analytics-pipeline.yml +++ b/docker-compose-analytics-pipeline.yml @@ -3,10 +3,14 @@ version: "2.1" services: namenode: image: edxops/analytics_pipeline_hadoop_namenode:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.namenode + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.namenode" hostname: namenode environment: - CLUSTER_NAME=devstack + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.namenode ports: - 127.0.0.1:50070:50070 command: ["/run.sh"] @@ -15,12 +19,16 @@ services: datanode: image: edxops/analytics_pipeline_hadoop_datanode:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.datanode + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.datanode" hostname: datanode environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" depends_on: - namenode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.datanode ports: - 127.0.0.1:50075:50075 command: ["/run.sh"] @@ -29,7 +37,7 @@ services: resourcemanager: image: edxops/analytics_pipeline_hadoop_resourcemanager:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.resourcemanager + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.resourcemanager" hostname: resourcemanager environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" @@ -40,13 +48,17 @@ services: depends_on: - namenode - datanode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.resourcemanager ports: - 127.0.0.1:8088:8088 # resource manager web ui command: ["/run.sh"] nodemanager: image: edxops/analytics_pipeline_hadoop_nodemanager:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.nodemanager + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.nodemanager" hostname: nodemanager environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" @@ -60,6 +72,10 @@ services: - resourcemanager - namenode - datanode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.nodemanager ports: - 127.0.0.1:8042:8042 # node manager web ui - 127.0.0.1:19888:19888 # node manager job history server ui @@ -67,8 +83,12 @@ services: sparkmaster: image: edxops/analytics_pipeline_spark_master:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.sparkmaster + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkmaster" hostname: sparkmaster + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.sparkmaster ports: - 127.0.0.1:8080:8080 - 127.0.0.1:7077:7077 # spark master port @@ -77,25 +97,33 @@ services: sparkworker: image: edxops/analytics_pipeline_spark_worker:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.sparkworker + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkworker" hostname: sparkworker depends_on: - sparkmaster environment: - SPARK_MASTER=spark://sparkmaster:7077 + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.sparkworker ports: - 127.0.0.1:8081:8081 # spark worker UI vertica: image: iamamr/vertica:9.1.0-0 hostname: vertica - container_name: edx.devstack.analytics_pipeline.vertica + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.vertica" volumes: - vertica_data:/home/dbadmin/docker + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.vertica analyticspipeline: image: edxops/analytics_pipeline:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline" hostname: analyticspipeline volumes: - ${DEVSTACK_WORKSPACE}/edx-analytics-pipeline:/edx/app/analytics_pipeline/analytics_pipeline @@ -109,6 +137,10 @@ services: - sparkworker - elasticsearch - vertica + networks: + default: + aliases: + - edx.devstack.analytics_pipeline ports: - 127.0.0.1:4040:4040 # spark web UI environment: diff --git a/docker-compose-marketing-site.yml b/docker-compose-marketing-site.yml index 4bd2693b7a..bc48b89ce1 100644 --- a/docker-compose-marketing-site.yml +++ b/docker-compose-marketing-site.yml @@ -7,7 +7,7 @@ services: - MARKETING_SITE_ROOT="http://localhost:8080" marketing: - container_name: edx.devstack.marketing + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.marketing" depends_on: - mysql - memcached @@ -26,5 +26,9 @@ services: # IP address of your machine to enable debugging (IP_ADDRESS set in .env file) - XDEBUG_CONFIG=remote_host=${XDEBUG_IP_ADDRESS:-127.0.0.1} image: edxops/edx-mktg:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.marketing ports: - "8080:80" diff --git a/docker-compose-watchers-nfs.yml b/docker-compose-watchers-nfs.yml index 532a49060f..16d5062a24 100644 --- a/docker-compose-watchers-nfs.yml +++ b/docker-compose-watchers-nfs.yml @@ -3,7 +3,7 @@ version: "2.1" services: lms_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.lms_watcher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.lms_watcher ASSET_WATCHER_TIMEOUT: 12 @@ -14,10 +14,14 @@ services: - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules - src-nfs:/edx/src - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes:cached + networks: + default: + aliases: + - edx.devstack.lms_watcher studio_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.studio_watcher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.studio_watcher ASSET_WATCHER_TIMEOUT: 12 @@ -28,6 +32,11 @@ services: - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules - src-nfs:/edx/src:cached - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes:cached + networks: + default: + aliases: + - edx.devstack.studio_watcher + volumes: edxapp_lms_assets: edxapp_studio_assets: diff --git a/docker-compose-watchers.yml b/docker-compose-watchers.yml index 7c7828f701..00a37f31d0 100644 --- a/docker-compose-watchers.yml +++ b/docker-compose-watchers.yml @@ -3,7 +3,7 @@ version: "2.1" services: lms_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.lms_watcher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.lms_watcher ASSET_WATCHER_TIMEOUT: 12 @@ -14,10 +14,14 @@ services: - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules - ${DEVSTACK_WORKSPACE}/src:/edx/src:cached - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes:cached + networks: + default: + aliases: + - edx.devstack.lms_watcher studio_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.studio_watcher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.studio_watcher ASSET_WATCHER_TIMEOUT: 12 @@ -28,6 +32,10 @@ services: - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules - ${DEVSTACK_WORKSPACE}/src:/edx/src:cached - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes:cached + networks: + default: + aliases: + - edx.devstack.studio_watcher volumes: edxapp_lms_assets: diff --git a/docker-compose-xqueue.yml b/docker-compose-xqueue.yml index bf4a73ac54..968221d858 100644 --- a/docker-compose-xqueue.yml +++ b/docker-compose-xqueue.yml @@ -2,19 +2,27 @@ version: "2.1" services: xqueue: - container_name: edx.devstack.xqueue + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue" image: edxops/xqueue:${OPENEDX_RELEASE:-latest} command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py runserver 0.0.0.0:18040 ; sleep 2; done' volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file + networks: + default: + aliases: + - edx.devstack.xqueue ports: - 18040:18040 xqueue_consumer: - container_name: edx.devstack.xqueue_consumer + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue_consumer" image: edxops/xqueue:${OPENEDX_RELEASE:-latest} command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py run_consumer ; sleep 2; done' volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached + networks: + default: + aliases: + - edx.devstack.xqueue_consumer # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file diff --git a/docker-compose.yml b/docker-compose.yml index d96017260e..01dd47c399 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,10 +17,14 @@ services: # ================================================ chrome: - container_name: edx.devstack.chrome + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.chrome" hostname: chrome.devstack.edx image: edxops/chrome:${OPENEDX_RELEASE:-latest} shm_size: 2g + networks: + default: + aliases: + - edx.devstack.chrome ports: - "15900:5900" volumes: # for file uploads @@ -28,18 +32,26 @@ services: - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data devpi: - container_name: edx.devstack.devpi + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.devpi" hostname: devpi.devstack.edx image: edxops/devpi:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.devpi ports: - "3141:3141" volumes: - devpi_data:/data elasticsearch: - container_name: edx.devstack.elasticsearch + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.elasticsearch" hostname: elasticsearch.devstack.edx image: edxops/elasticsearch:devstack + networks: + default: + aliases: + - edx.devstack.elasticsearch # TODO: What to do about these forwarded ports? They'll conflict with ports forwarded by the Vagrant VM. # ports: # - "9200:9200" @@ -50,15 +62,23 @@ services: # This is meant to be used to test ES upgrades so that we do not have to upgrade all of our services to ES5 at once. elasticsearch-5: - container_name: edx.devstack.elasticsearch-5 + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.elasticsearch-5" hostname: elasticsearch-5.devstack.edx image: elasticsearch:5.6.16 + networks: + default: + aliases: + - edx.devstack.elasticsearch-5 firefox: - container_name: edx.devstack.firefox + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.firefox" hostname: firefox.devstack.edx image: edxops/firefox:${OPENEDX_RELEASE:-latest} shm_size: 2g + networks: + default: + aliases: + - edx.devstack.firefox ports: - "25900:5900" volumes: # for file uploads @@ -66,9 +86,13 @@ services: - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data memcached: - container_name: edx.devstack.memcached + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.memcached" hostname: memcached.devstack.edx image: memcached:1.5.10-alpine + networks: + default: + aliases: + - edx.devstack.memcached # ports: # - "11211:11211" @@ -77,9 +101,13 @@ services: # to conserve disk space, and disable the journal for a minor performance gain. # See https://docs.mongodb.com/v3.0/reference/program/mongod/#options for complete details. command: mongod --smallfiles --nojournal --storageEngine wiredTiger - container_name: edx.devstack.mongo + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mongo" hostname: mongo.devstack.edx image: mongo:${MONGO_VERSION:-3.6.17} + networks: + default: + aliases: + - edx.devstack.mongo # ports: # - "27017:27017" volumes: @@ -87,12 +115,16 @@ services: mysql: command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci - container_name: edx.devstack.mysql + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql" hostname: mysql.devstack.edx environment: MYSQL_ROOT_PASSWORD: "" MYSQL_ALLOW_EMPTY_PASSWORD: "yes" image: mysql:5.6 + networks: + default: + aliases: + - edx.devstack.mysql # ports: # - "3506:3306" volumes: @@ -100,22 +132,30 @@ services: mysql57: command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci - container_name: edx.devstack.mysql57 + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql57" hostname: mysql57.devstack.edx environment: MYSQL_ROOT_PASSWORD: "" MYSQL_ALLOW_EMPTY_PASSWORD: "yes" image: mysql:5.7 + networks: + default: + aliases: + - edx.devstack.mysql57 # ports: # - "3506:3306" volumes: - mysql57_data:/var/lib/mysql redis: - container_name: edx.devstack.redis + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.redis" hostname: redis.devstack.edx image: redis:2.8 command: redis-server --requirepass password + networks: + default: + aliases: + - edx.devstack.redis # ================================================ # edX services @@ -123,7 +163,7 @@ services: credentials: command: bash -c 'source /edx/app/credentials/credentials_env && while true; do python /edx/app/credentials/credentials/manage.py runserver 0.0.0.0:18150; sleep 2; done' - container_name: edx.devstack.credentials + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.credentials" hostname: credentials.devstack.edx depends_on: - mysql @@ -139,12 +179,16 @@ services: ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/credentials:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.credentials ports: - "18150:18150" discovery: command: bash -c 'source /edx/app/discovery/discovery_env && while true; do python /edx/app/discovery/discovery/manage.py runserver 0.0.0.0:18381; sleep 2; done' - container_name: edx.devstack.discovery + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.discovery" hostname: discovery.devstack.edx depends_on: - mysql @@ -158,6 +202,10 @@ services: ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/discovery:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.discovery ports: - "18381:18381" volumes: @@ -165,7 +213,7 @@ services: ecommerce: command: bash -c 'source /edx/app/ecommerce/ecommerce_env && while true; do python /edx/app/ecommerce/ecommerce/manage.py runserver 0.0.0.0:18130; sleep 2; done' - container_name: edx.devstack.ecommerce + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.ecommerce" hostname: ecommerce.devstack.edx depends_on: - mysql @@ -179,18 +227,26 @@ services: DJANGO_WATCHMAN_TIMEOUT: 30 ENABLE_DJANGO_TOOLBAR: 1 image: edxops/ecommerce:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.ecommerce ports: - "18130:18130" edx_notes_api: command: bash -c 'source /edx/app/edx_notes_api/edx_notes_api_env && while true; do python /edx/app/edx_notes_api/edx_notes_api/manage.py runserver 0.0.0.0:18120 --settings notesserver.settings.devstack; sleep 2; done' - container_name: edx.devstack.edxnotesapi + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.edxnotesapi" hostname: edx_notes_api.devstack.edx depends_on: - devpi - elasticsearch - mysql image: edxops/notes:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.edxnotesapi ports: - "18120:18120" environment: @@ -206,19 +262,23 @@ services: forum: command: bash -c 'source /edx/app/forum/ruby_env && source /edx/app/forum/devstack_forum_env && cd /edx/app/forum/cs_comments_service && bundle install && while true; do ruby app.rb -o 0.0.0.0 ; sleep 2; done' - container_name: edx.devstack.forum + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.forum" hostname: forum.devstack.edx depends_on: - mongo - memcached - elasticsearch image: edxops/forum:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.forum ports: - "44567:4567" lms: command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack_docker; sleep 2; done' - container_name: edx.devstack.lms + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms" hostname: lms.devstack.edx depends_on: - devpi @@ -240,6 +300,10 @@ services: NO_PYTHON_UNINSTALL: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.lms ports: - "18000:18000" - "19876:19876" # JS test debugging @@ -250,7 +314,7 @@ services: registrar: command: bash -c 'source /edx/app/registrar/registrar_env && while true; do python /edx/app/registrar/registrar/manage.py runserver 0.0.0.0:18734; sleep 2; done' - container_name: edx.devstack.registrar + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.registrar" hostname: registrar.devstack.edx depends_on: - discovery @@ -278,6 +342,10 @@ services: CELERY_BROKER_PASSWORD: password DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/registrar:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.registrar ports: - "18734:18734" volumes: @@ -285,7 +353,7 @@ services: registrar-worker: command: bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && celery -A registrar worker -l debug -c 2' - container_name: edx.devstack.registrar-worker + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.registrar-worker" hostname: registrar-worker.devstack.edx depends_on: - lms @@ -308,6 +376,10 @@ services: CELERY_BROKER_PASSWORD: password DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/registrar:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.registrar-worker ports: - "18735:18735" volumes: @@ -315,7 +387,7 @@ services: studio: command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py cms runserver 0.0.0.0:18010 --settings devstack_docker; sleep 2; done' - container_name: edx.devstack.studio + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio" hostname: studio.devstack.edx depends_on: - devpi @@ -336,6 +408,10 @@ services: NO_PYTHON_UNINSTALL: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.studio ports: - "18010:18010" - "19877:19877" # JS test debugging @@ -357,7 +433,11 @@ services: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/frontend-app-publisher' - container_name: edx.devstack.frontend-app-publisher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-publisher" + networks: + default: + aliases: + - edx.devstack.frontend-app-publisher ports: - "18400:18400" depends_on: @@ -368,7 +448,11 @@ services: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/gradebook' - container_name: edx.devstack.gradebook + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.gradebook" + networks: + default: + aliases: + - edx.devstack.gradebook ports: - "1994:1994" depends_on: @@ -379,7 +463,11 @@ services: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/program-console' - container_name: edx.devstack.program-console + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.program-console" + networks: + default: + aliases: + - edx.devstack.program-console ports: - "1976:1976" depends_on: @@ -391,7 +479,11 @@ services: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/frontend-app-learning' - container_name: edx.devstack.frontend-app-learning + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-learning" + networks: + default: + aliases: + - edx.devstack.frontend-app-learning ports: - "2000:2000" depends_on: diff --git a/docs/devpi.rst b/docs/devpi.rst index dc1533b950..b90ae837cd 100644 --- a/docs/devpi.rst +++ b/docs/devpi.rst @@ -60,6 +60,6 @@ Monitoring devpi ---------------- You can monitor the devpi logs by running this command on the host: -``docker logs -f edx.devstack.devpi`` or looking at the output in +``make devpi-logs`` or looking at the output in Kitematic. You can also check the devpi server status by visiting: http://localhost:3141/+status diff --git a/gather-feature-toggle-state.sh b/gather-feature-toggle-state.sh old mode 100644 new mode 100755 index 2ed9946195..451d3089b8 --- a/gather-feature-toggle-state.sh +++ b/gather-feature-toggle-state.sh @@ -12,20 +12,24 @@ if [ -e feature-toggle-data ]; then fi mkdir feature-toggle-data -docker exec -t edx.devstack.credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py dumpdata waffle --format=json > /edx/app/credentials/credentials/credentials_waffle.json' -docker cp edx.devstack.credentials:/edx/app/credentials/credentials/credentials_waffle.json feature-toggle-data -docker exec -t edx.devstack.credentials bash -c 'rm /edx/app/credentials/credentials/credentials_waffle.json' +credentials_id="$(make --silent dev.print-container.credentials)" +docker exec -t "$credentials_id" bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py dumpdata waffle --format=json > /edx/app/credentials/credentials/credentials_waffle.json' +docker cp "$credentials_id":/edx/app/credentials/credentials/credentials_waffle.json feature-toggle-data +docker exec -t "$credentials_id" bash -c 'rm /edx/app/credentials/credentials/credentials_waffle.json' -docker exec -t edx.devstack.discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py dumpdata waffle --format=json > /edx/app/discovery/discovery/discovery_waffle.json' -docker cp edx.devstack.discovery:/edx/app/discovery/discovery/discovery_waffle.json feature-toggle-data -docker exec -t edx.devstack.discovery bash -c '> /edx/app/discovery/discovery/discovery_waffle.json' +discovery_id="$(make --silent dev.print-container.discovery)" +docker exec -t "$discovery_id" bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py dumpdata waffle --format=json > /edx/app/discovery/discovery/discovery_waffle.json' +docker cp "$discovery_id":/edx/app/discovery/discovery/discovery_waffle.json feature-toggle-data +docker exec -t "$discovery_id" bash -c '> /edx/app/discovery/discovery/discovery_waffle.json' -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py dumpdata waffle --format=json > /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' -docker cp edx.devstack.ecommerce:/edx/app/ecommerce/ecommerce/ecommerce_waffle.json feature-toggle-data -docker exec -t edx.devstack.ecommerce bash -c 'rm /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' +ecommerce_id="$(make --silent dev.print-container.ecommerce)" +docker exec -t "$ecommerce_id" bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py dumpdata waffle --format=json > /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' +docker cp "$ecommerce_id":/edx/app/ecommerce/ecommerce/ecommerce_waffle.json feature-toggle-data +docker exec -t "$ecommerce_id" bash -c 'rm /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle --format=json > /edx/app/edxapp/edx-platform/lms_waffle.json' -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle_utils --format=json > /edx/app/edxapp/edx-platform/lms_waffle_utils.json' -docker cp edx.devstack.lms:/edx/app/edxapp/edx-platform/lms_waffle.json feature-toggle-data -docker cp edx.devstack.lms:/edx/app/edxapp/edx-platform/lms_waffle_utils.json feature-toggle-data -docker exec -t edx.devstack.lms bash -c 'rm /edx/app/edxapp/edx-platform/lms_waffle*.json' +lms_id="$(make --silent dev.print-container.lms)" +docker exec -t "$lms_id" bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle --format=json > /edx/app/edxapp/edx-platform/lms_waffle.json' +docker exec -t "$lms_id" bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle_utils --format=json > /edx/app/edxapp/edx-platform/lms_waffle_utils.json' +docker cp "$lms_id":/edx/app/edxapp/edx-platform/lms_waffle.json feature-toggle-data +docker cp "$lms_id":/edx/app/edxapp/edx-platform/lms_waffle_utils.json feature-toggle-data +docker exec -t "$lms_id" bash -c 'rm /edx/app/edxapp/edx-platform/lms_waffle*.json' diff --git a/in b/in index 184e21ed1c..e1aa6d8946 100755 --- a/in +++ b/in @@ -1,7 +1,26 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash +# Run a command in a service's container. +# Try to bring up the service if it isn't already up. +# Example: +# ~/devstack> ./in registrar ls /edx/app/ +# edx_ansible nginx registrar supervisor +# Example 2: +# ~/devstack> ./in registrar "cd /edx/app/registrar && ls" +# data registrar registrar.sh venvs +# devstack.sh registrar_env registrar-workers.sh +# nodeenvs registrar_gunicorn.py staticfiles -SERVICE=$1 +set -e +set -o pipefail +set -u + +service="$1" shift -export DEVSTACK_WORKSPACE=${DEVSTACK_WORKSPACE:-$(pwd)/..} -docker-compose -f docker-compose.yml -f docker-compose-host.yml -f docker-compose-themes.yml run $SERVICE exec "$@" +container_id=$(make --silent dev.print-container."$service") +if [[ -z "$container_id" ]]; then + make --silent dev.up."$service" + container_id=$(make --silent dev.print-container."$service") +fi + +docker exec -it "$container_id" bash -c "$*" diff --git a/load-db.sh b/load-db.sh index 24b70fada5..9b9aaa6c75 100755 --- a/load-db.sh +++ b/load-db.sh @@ -17,5 +17,6 @@ then fi echo "Loading the $1 database..." -docker exec -i edx.devstack.mysql mysql -uroot $1 < $1.sql +mysql_container=$(make -s dev.print-container.mysql) +docker exec "$mysql_container" bash -c "mysql -uroot $1" < $1.sql echo "Finished loading the $1 database!" diff --git a/marketing.mk b/marketing.mk index 16a8adb4a2..c1496c3ef0 100644 --- a/marketing.mk +++ b/marketing.mk @@ -3,7 +3,7 @@ help-marketing: ## Display this help message @awk -F ':.*?## ' '/^[a-zA-Z]/ && NF==2 {printf "\033[36m %-25s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | grep marketing | sort marketing-shell: ## Run a shell on the marketing site container - docker exec -it edx.devstack.marketing env TERM=$(TERM) bash -c 'cd /edx/app/edx-mktg/edx-mktg; exec /bin/bash -sh' + docker-compose exec marketing env TERM=$(TERM) bash -c 'cd /edx/app/edx-mktg/edx-mktg; exec /bin/bash -sh' stop-marketing: ## Stop all services (including the marketing site) with host volumes docker-compose -f docker-compose.yml -f docker-compose-host.yml -f docker-compose-themes.yml -f docker-compose-marketing-site.yml -f docker-compose-marketing-site-host.yml stop diff --git a/options.mk b/options.mk index 5daa0ac414..1c63c0d1bd 100644 --- a/options.mk +++ b/options.mk @@ -10,8 +10,12 @@ DEVSTACK_WORKSPACE ?= $(shell pwd)/.. # Name of Docker Compose project. # See https://docs.docker.com/compose/reference/envvars/#compose_project_name -# Defaults to 'devstack'. -COMPOSE_PROJECT_NAME ?= devstack +# Defaults to 'devstack' should OPENEDX_RELEASE not be defined. +ifdef OPENEDX_RELEASE + COMPOSE_PROJECT_NAME ?= devstack-${OPENEDX_RELEASE} +else + COMPOSE_PROJECT_NAME ?= devstack +endif # increase Docker Compose HTTP timeout so that devstack provisioning does not fail in unstable networks COMPOSE_HTTP_TIMEOUT=180 diff --git a/provision-analyticspipeline.sh b/provision-analyticspipeline.sh index 70e458e683..6ee97e2733 100755 --- a/provision-analyticspipeline.sh +++ b/provision-analyticspipeline.sh @@ -17,19 +17,19 @@ elif [ ! -d "$DEVSTACK_WORKSPACE" ]; then fi # Bring the pipeline containers online. -docker-compose $DOCKER_COMPOSE_FILES up -d analyticspipeline +docker-compose up -d analyticspipeline # Analytics pipeline has dependency on lms but we only need its db schema & not full lms. So we'll just load their db # schemas as part of analytics pipeline provisioning. If there is a need of a fully fledged LMS, then provision lms # by following their documentation. -if [[ ! -z "`docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='edxapp'" 2>&1`" ]]; +if [[ ! -z "`docker-compose exec mysql bash -c "mysql -uroot -se \"SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='edxapp'\"" 2>&1`" ]]; then echo -e "${GREEN}LMS DB exists, skipping lms schema load.${NC}" else echo -e "${GREEN}LMS DB not found, provisioning lms schema.${NC}" - docker exec -i edx.devstack.mysql mysql -uroot mysql < provision.sql + docker-compose exec -T mysql bash -c 'mysql -uroot mysql' < provision.sql ./load-db.sh edxapp - docker-compose $DOCKER_COMPOSE_FILES up -d lms studio + docker-compose up -d lms studio docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' #Installing prereqs crashes the process docker-compose restart lms @@ -39,11 +39,11 @@ fi echo -e "${GREEN}LMS database provisioned successfully...${NC}" echo -e "${GREEN}Creating databases and users...${NC}" -docker exec -i edx.devstack.mysql mysql -uroot mysql < provision-analytics-pipeline.sql +docker-compose exec -T mysql bash -c 'mysql -uroot mysql' < provision-analytics-pipeline.sql # initialize hive metastore echo -e "${GREEN}Initializing HIVE metastore...${NC}" -docker-compose $DOCKER_COMPOSE_FILES exec analyticspipeline bash -c '/edx/app/hadoop/hive/bin/schematool -dbType mysql -initSchema' +docker-compose exec analyticspipeline bash -c '/edx/app/hadoop/hive/bin/schematool -dbType mysql -initSchema' # materialize hadoop directory structure echo -e "${GREEN}Initializing Hadoop directory structure...${NC}" @@ -56,4 +56,4 @@ done sleep 10 # for datanode & other services to activate echo -e "${GREEN}Namenode is ready!${NC}" -docker exec -u hadoop -i edx.devstack.analytics_pipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' +docker-compose exec -u hadoop analyticspipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' diff --git a/provision-credentials.sh b/provision-credentials.sh index 32fccdaf61..a75044dd60 100755 --- a/provision-credentials.sh +++ b/provision-credentials.sh @@ -6,23 +6,23 @@ name=credentials port=18150 -docker-compose $DOCKER_COMPOSE_FILES up -d $name +docker-compose up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" echo -e "${GREEN}Configuring site for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' +docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker-compose exec ${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-e2e.sh b/provision-e2e.sh index b9e7b5508e..86722225d1 100755 --- a/provision-e2e.sh +++ b/provision-e2e.sh @@ -10,7 +10,7 @@ elif [ ! -d "$DEVSTACK_WORKSPACE" ]; then fi # Copy the test course tarball into the studio container -docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz edx.devstack.studio:/tmp/ +docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz "$(make --silent dev.print-container.studio)":/tmp/ # Extract the test course tarball docker-compose exec -T studio bash -c 'cd /tmp && tar xzf course.tar.gz' diff --git a/provision-ecommerce.sh b/provision-ecommerce.sh index c1680fab02..75353fcd6f 100755 --- a/provision-ecommerce.sh +++ b/provision-ecommerce.sh @@ -4,6 +4,6 @@ ./provision-ida.sh ecommerce ecommerce 18130 # Configure ecommerce -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/' -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' +docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/' +docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' +docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' diff --git a/provision-forum.sh b/provision-forum.sh index 6196ac0176..afc51e2827 100755 --- a/provision-forum.sh +++ b/provision-forum.sh @@ -2,5 +2,5 @@ set -e set -o pipefail set -x -docker-compose $DOCKER_COMPOSE_FILES up -d forum +docker-compose up -d forum docker-compose exec -T forum bash -c 'source /edx/app/forum/ruby_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/' diff --git a/provision-ida-user.sh b/provision-ida-user.sh index 1a3b93c9ee..965de8a566 100755 --- a/provision-ida-user.sh +++ b/provision-ida-user.sh @@ -7,8 +7,8 @@ client_port=$3 echo -e "${GREEN}Creating service user and OAuth2 applications for ${app_name}...${NC}" # Create the service user. -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" # Create the DOT applications - one for single sign-on and one for backend service IDA-to-IDA authentication. -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "http://localhost:$3/complete/edx-oauth2/" --client-id "$1-sso-key" --client-secret "$1-sso-secret" --scopes "user_id" $1-sso $1_worker' -- "$app_name" "$client_name" "$client_port" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "$1-backend-service-key" --client-secret "$1-backend-service-secret" $1-backend-service $1_worker' -- "$app_name" "$client_name" "$client_port" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "http://localhost:$3/complete/edx-oauth2/" --client-id "$1-sso-key" --client-secret "$1-sso-secret" --scopes "user_id" $1-sso $1_worker' -- "$app_name" "$client_name" "$client_port" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "$1-backend-service-key" --client-secret "$1-backend-service-secret" $1-backend-service $1_worker' -- "$app_name" "$client_name" "$client_port" diff --git a/provision-ida.sh b/provision-ida.sh index 62bbf4d0ad..c830ad2c78 100755 --- a/provision-ida.sh +++ b/provision-ida.sh @@ -4,20 +4,20 @@ client_name=$2 # The name of the Oauth client stored in the edxapp DB. client_port=$3 # The port corresponding to this IDA service in devstack. container_name=${4:-$1} # (Optional) The name of the container. If missing, will use app_name. -docker-compose $DOCKER_COMPOSE_FILES up -d $app_name +docker-compose up -d $app_name echo -e "${GREEN}Installing requirements for ${app_name}...${NC}" -docker exec -t edx.devstack.${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" +docker-compose exec ${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" echo -e "${GREEN}Running migrations for ${app_name}...${NC}" -docker exec -t edx.devstack.${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" +docker-compose exec ${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" echo -e "${GREEN}Creating super-user for ${app_name}...${NC}" -docker exec -t edx.devstack.${container_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" +docker-compose exec ${container_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" ./provision-ida-user.sh $app_name $client_name $client_port # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${app_name}...${NC}" -docker exec -t edx.devstack.${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" +docker-compose exec ${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" diff --git a/provision-lms.sh b/provision-lms.sh index 7ef1b8e252..6ec0221867 100755 --- a/provision-lms.sh +++ b/provision-lms.sh @@ -10,7 +10,7 @@ apps=( lms studio ) # Bring edxapp containers online for app in "${apps[@]}"; do - docker-compose $DOCKER_COMPOSE_FILES up -d $app + docker-compose up -d $app done docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' diff --git a/provision-marketing.sh b/provision-marketing.sh index 48bf3f88ca..b7e4417550 100755 --- a/provision-marketing.sh +++ b/provision-marketing.sh @@ -7,7 +7,7 @@ set -x YELLOW='\033[1;33m' NC='\033[0m' # No Color -docker-compose $DOCKER_COMPOSE_FILES up -d marketing +docker-compose up -d marketing set +x echo -e "${YELLOW}edX Marketing Site is not fully provisioned yet.${NC}" diff --git a/provision-notes.sh b/provision-notes.sh index a840408741..12ff3767fb 100755 --- a/provision-notes.sh +++ b/provision-notes.sh @@ -1,8 +1,8 @@ # Provisioning script for the notes service # Common provisioning tasks for IDAs, including requirements, migrations, oauth client creation, etc. -./provision-ida.sh edx_notes_api edx-notes 18120 edxnotesapi +./provision-ida.sh edx_notes_api edx-notes 18120 edx_notes_api # This will build the elasticsearch index for notes. echo -e "${GREEN}Creating indexes for edx_notes_api...${NC}" -docker exec -t edx.devstack.edxnotesapi bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py rebuild_index --noinput' -- edx_notes_api +docker-compose exec edx_notes_api bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py rebuild_index --noinput' -- edx_notes_api diff --git a/provision-registrar.sh b/provision-registrar.sh index 3fd78f683c..88d2f6d129 100755 --- a/provision-registrar.sh +++ b/provision-registrar.sh @@ -3,20 +3,20 @@ name=registrar port=18734 -docker-compose $DOCKER_COMPOSE_FILES up -d $name +docker-compose up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make requirements' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make migrate' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make createsuperuser' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make createsuperuser' -- "$name" ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c ' if ! source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make static 2>registrar_make_static.err; then echo "------- Last 100 lines of stderr"; tail regsitrar_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker-compose exec ${name} bash -c ' if ! source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make static 2>registrar_make_static.err; then echo "------- Last 100 lines of stderr"; tail regsitrar_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-retirement-user.sh b/provision-retirement-user.sh index b3a5dac2e3..3267258628 100755 --- a/provision-retirement-user.sh +++ b/provision-retirement-user.sh @@ -4,5 +4,5 @@ app_name=$1 user_name=$2 echo -e "${GREEN}Creating retirement service user ${user_name} and DOT Application ${app_name}...${NC}" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" diff --git a/provision-xqueue.sh b/provision-xqueue.sh index 13e0d0869b..7736394a0a 100755 --- a/provision-xqueue.sh +++ b/provision-xqueue.sh @@ -3,11 +3,11 @@ set -o pipefail set -x # Bring up XQueue, we don't need the consumer for provisioning -docker-compose $DOCKER_COMPOSE_FILES up -d xqueue +docker-compose up -d xqueue # Update dependencies -docker-compose $DOCKER_COMPOSE_FILES exec xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && make requirements' +docker-compose exec xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && make requirements' # Run migrations -docker-compose $DOCKER_COMPOSE_FILES exec xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py migrate' +docker-compose exec xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py migrate' # Add users that graders use to fetch data, there's one default user in Ansible which is part of our settings -docker-compose $DOCKER_COMPOSE_FILES exec xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py update_users' +docker-compose exec xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py update_users' diff --git a/provision.sh b/provision.sh index 60e6e04bfe..42761b93c3 100755 --- a/provision.sh +++ b/provision.sh @@ -122,7 +122,7 @@ fi # Ensure the MySQL server is online and usable echo "${GREEN}Waiting for MySQL.${NC}" -until docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')" &> /dev/null +until docker-compose exec -T mysql bash -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 @@ -136,20 +136,20 @@ echo -e "${GREEN}MySQL ready.${NC}" # Ensure that the MySQL databases and users are created for all IDAs. # (A no-op for databases and users that already exist). echo -e "${GREEN}Ensuring MySQL databases and users exist...${NC}" -docker exec -i edx.devstack.mysql mysql -uroot mysql < provision.sql +docker-compose exec -T mysql bash -c "mysql -uroot mysql" < provision.sql # If necessary, ensure the MongoDB server is online and usable # and create its users. if needs_mongo "$to_provision"; then echo -e "${GREEN}Waiting for MongoDB...${NC}" - until docker exec -i edx.devstack.mongo mongo --eval "printjson(db.serverStatus())" &> /dev/null + until docker-compose exec -T mongo bash -c 'mongo --eval "printjson(db.serverStatus())"' &> /dev/null do printf "." sleep 1 done echo -e "${GREEN}MongoDB ready.${NC}" echo -e "${GREEN}Creating MongoDB users...${NC}" - docker exec -i edx.devstack.mongo mongo < mongo-provision.js + docker-compose exec -T mongo bash -c "mongo" < mongo-provision.js else echo -e "${GREEN}MongoDB preparation not required; skipping.${NC}" fi diff --git a/update-dbs-init-sql-scripts.sh b/update-dbs-init-sql-scripts.sh index 53d9830ee0..456e109a5e 100755 --- a/update-dbs-init-sql-scripts.sh +++ b/update-dbs-init-sql-scripts.sh @@ -24,8 +24,7 @@ make dev.clone.ssh make dev.provision.services.lms+ecommerce # dump schema and data from mysql databases in the mysql docker container and copy them to current directory in docker host -MYSQL_DOCKER_COMPOSE_SERVICE="mysql" -MYSQL_DOCKER_CONTAINER="edx.devstack.${MYSQL_DOCKER_COMPOSE_SERVICE}" +MYSQL_DOCKER_CONTAINER="$(make --silent dev.print-container.mysql)" for DB_NAME in "${DBS[@]}"; do DB_CREATION_SQL_SCRIPT="${DB_NAME}.sql" if [[ " ${EDXAPP_DBS[@]} " =~ " ${DB_NAME} " ]]; then diff --git a/upgrade_mongo_3_6.sh b/upgrade_mongo_3_6.sh index 1d0ce7c9c2..3644a8c7c9 100755 --- a/upgrade_mongo_3_6.sh +++ b/upgrade_mongo_3_6.sh @@ -10,12 +10,13 @@ NC='\033[0m' # No Color export MONGO_VERSION=3.4.24 echo -e "${GREEN}Sarting Mongo ${MONGO_VERSION}${NC}" -docker-compose up -d mongo +make dev.up.mongo +mongo_container="$(make -s dev.print-container.mongo)" echo -e "${GREEN}Waiting for MongoDB...${NC}" -until docker exec -i edx.devstack.mongo mongo --eval "printjson(db.serverStatus())" &> /dev/null +until docker exec "$mongo_container" bash -c 'mongo --eval \"printjson(db.serverStatus())\"' &> /dev/null do - if docker-compose logs mongo | grep -q "BadValue: Invalid value for version, found 3.6, expected '3.4' or '3.2'"; then + if docker logs "$mongo_container" | grep -q "BadValue: Invalid value for version, found 3.6, expected '3.4' or '3.2'"; then echo -e "${YELLOW}Already upgraded to Mongo 3.6, exiting${NC}" exit fi @@ -24,15 +25,15 @@ do done echo -e "${GREEN}MongoDB ready.${NC}" -MONGO_VERSION_LIVE=$(docker-compose exec mongo mongo --quiet --eval "printjson(db.version())") -MONGO_VERSION_COMPAT=$(docker-compose exec mongo mongo --quiet \ +MONGO_VERSION_LIVE=$(docker exec -it "$mongo_container" mongo --quiet --eval "printjson(db.version())") +MONGO_VERSION_COMPAT=$(docker exec -it "$mongo_container" mongo --quiet \ --eval "printjson(db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )['featureCompatibilityVersion'])") echo -e "${GREEN}Mongo Server version: ${MONGO_VERSION_LIVE}${NC}" echo -e "${GREEN}Mongo FeatureCompatibilityVersion version: ${MONGO_VERSION_COMPAT}${NC}" if echo "${MONGO_VERSION_COMPAT}" | grep -q "3\.2" ; then echo -e "${GREEN}Upgrading FeatureCompatibilityVersion to 3.4${NC}" - docker-compose exec mongo mongo --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"3.4\" } )" + docker exec -it "$mongo_container" mongo --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"3.4\" } )" else echo -e "${GREEN}FeatureCompatibilityVersion already set to 3.4${NC}" fi @@ -42,25 +43,26 @@ export MONGO_VERSION=3.6.17 echo echo -e "${GREEN}Restarting Mongo on version ${MONGO_VERSION}${NC}" -docker-compose up -d mongo +make dev.up.mongo +mongo_container="$(make -s dev.print-container.mongo)" echo -e "${GREEN}Waiting for MongoDB...${NC}" -until docker exec -i edx.devstack.mongo mongo --eval "printjson(db.serverStatus())" &> /dev/null +until docker exec "$mongo_container" bash -c 'mongo --eval \"printjson(db.serverStatus())\"' &> /dev/null do printf "." sleep 1 done echo -e "${GREEN}MongoDB ready.${NC}" -MONGO_VERSION_LIVE=$(docker-compose exec mongo mongo --quiet --eval "printjson(db.version())") -MONGO_VERSION_COMPAT=$(docker-compose exec mongo mongo --quiet \ +MONGO_VERSION_LIVE=$(docker exec -it "$mongo_container" mongo --quiet --eval "printjson(db.version())") +MONGO_VERSION_COMPAT=$(docker exec -it "$mongo_container" mongo --quiet \ --eval "printjson(db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )['featureCompatibilityVersion'])") echo -e "${GREEN}Mongo Server version: ${MONGO_VERSION_LIVE}${NC}" echo -e "${GREEN}Mongo FeatureCompatibilityVersion version: ${MONGO_VERSION_COMPAT}${NC}" if echo "${MONGO_VERSION_COMPAT}" | grep -q "3\.4" ; then echo -e "${GREEN}Upgrading FeatureCompatibilityVersion to 3.6${NC}" - docker-compose exec mongo mongo --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"3.6\" } )" + docker exec -it "$mongo_container" mongo --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"3.6\" } )" else echo -e "${GREEN}FeatureCompatibilityVersion already set to 3.6${NC}" fi