From ed62e56fd9b03ea08cfb8a7321d65f0750c23f55 Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Tue, 5 May 2020 19:26:38 -0400 Subject: [PATCH 01/11] Update `docker` commands to be `docker-compose` to handle unnamed containers for multi-version devstack support --- Makefile | 70 ++++++++-------- Makefile.edx | 10 +-- course-generator/create-courses.sh | 8 +- docker-compose-analytics-pipeline.yml | 40 +++++++-- docker-compose-marketing-site.yml | 5 +- docker-compose-watchers-nfs.yml | 10 ++- docker-compose-watchers.yml | 10 ++- docker-compose-xqueue.yml | 10 ++- docker-compose.yml | 114 ++++++++++++++++++++------ gather-feature-toggle-state.sh | 28 +++---- load-db.sh | 3 +- marketing.mk | 2 +- options.mk | 8 +- provision-analyticspipeline.sh | 8 +- provision-credentials.sh | 10 +-- provision-e2e.sh | 2 +- provision-ecommerce.sh | 6 +- provision-ida-user.sh | 6 +- provision-ida.sh | 8 +- provision-notes.sh | 4 +- provision-registrar.sh | 8 +- provision-retirement-user.sh | 4 +- provision.sh | 8 +- upgrade_mongo_3_6.sh | 4 +- 24 files changed, 251 insertions(+), 135 deletions(-) diff --git a/Makefile b/Makefile index 1134c14c5e..a4542e1e4f 100644 --- a/Makefile +++ b/Makefile @@ -266,93 +266,93 @@ validate: ## Validate the devstack configuration docker-compose config backup: dev.up.mysql+mongo+elasticsearch ## Write all data volumes to the host. - docker run --rm --volumes-from edx.devstack.mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql - docker run --rm --volumes-from edx.devstack.mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db - docker run --rm --volumes-from edx.devstack.elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data + docker-compose run --rm --volumes-from -T mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql + docker-compose run --rm --volumes-from -T mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db + docker-compose run --rm --volumes-from -T elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data restore: dev.up.mysql+mongo+elasticsearch ## Restore all data volumes from the host. WARNING: THIS WILL OVERWRITE ALL EXISTING DATA! - docker run --rm --volumes-from edx.devstack.mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz - docker run --rm --volumes-from edx.devstack.mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz - docker run --rm --volumes-from edx.devstack.elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz + docker-compose run --rm --volumes-from -T mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz + docker-compose run --rm --volumes-from -T mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz + docker-compose run --rm --volumes-from -T elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz # TODO: Print out help for this target. Even better if we can iterate over the # services in docker-compose.yml, and print the actual service names. %-shell: ## Run a shell on the specified service container - docker exec -it edx.devstack.$* /bin/bash + docker-compose exec $* /bin/bash analyticspipeline-shell: ## Run a shell on the analytics pipeline container - docker exec -it edx.devstack.analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open + docker-compose exec analyticspipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open credentials-shell: ## Run a shell on the credentials container - docker exec -it edx.devstack.credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' + docker-compose exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' discovery-shell: ## Run a shell on the discovery container - docker exec -it edx.devstack.discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open + docker-compose exec discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open ecommerce-shell: ## Run a shell on the ecommerce container - docker exec -it edx.devstack.ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open + docker-compose exec ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open e2e-shell: ## Start the end-to-end tests container with a shell - docker run -it --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash + docker-compose run --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash registrar-shell: ## Run a shell on the registrar site container - docker exec -it edx.devstack.registrar env TERM=$(TERM) /edx/app/registrar/devstack.sh open + docker-compose exec registrar env TERM=$(TERM) /edx/app/registrar/devstack.sh open %-update-db: ## Run migrations for the specified service container - docker exec -t edx.devstack.$* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' + docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' studio-update-db: ## Run migrations for the Studio container - docker exec -t edx.devstack.studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' + docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' lms-update-db: ## Run migrations LMS container - docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' + docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' update-db: | $(DB_MIGRATION_TARGETS) ## Run the migrations for DEFAULT_SERVICES forum-restart-devserver: ## Kill the forum's Sinatra development server. The watcher process will restart it. - docker exec -t edx.devstack.forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose exec forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")' lms-shell: ## Run a shell on the LMS container - docker exec -it edx.devstack.lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose exec lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open lms-watcher-shell: ## Run a shell on the LMS watcher container - docker exec -it edx.devstack.lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose exec lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open %-attach: ## Attach to the specified service container process to use the debugger & see logs. - docker attach edx.devstack.$* + docker-compose attach $* lms-restart: lms-restart-devserver studio-shell: ## Run a shell on the Studio container - docker exec -it edx.devstack.studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose exec studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-watcher-shell: ## Run a shell on the studio watcher container - docker exec -it edx.devstack.studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose exec studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-restart: studio-restart-devserver xqueue-shell: ## Run a shell on the XQueue container - docker exec -it edx.devstack.xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open + docker-compose exec xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue-restart: xqueue-restart-devserver %-restart-devserver: ## Kill a service's Django development server. The watcher process should restart it. - docker exec -t edx.devstack.$* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose exec $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' xqueue_consumer-shell: ## Run a shell on the XQueue consumer container - docker exec -it edx.devstack.xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open + docker-compose exec xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue_consumer-restart: ## Kill the XQueue development server. The watcher process will restart it. - docker exec -t edx.devstack.xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose exec xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")' %-static: ## Rebuild static assets for the specified service container - docker exec -t edx.devstack.$* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' + docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' lms-static: ## Rebuild static assets for the LMS container - docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets lms' + docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets lms' studio-static: ## Rebuild static assets for the Studio container - docker exec -t edx.devstack.studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets studio' + docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets studio' static: | credentials-static discovery-static ecommerce-static lms-static studio-static ## Rebuild static assets for all service containers @@ -361,16 +361,16 @@ healthchecks: dev.check.$(DEFAULT_SERVICES) healthchecks.%: dev.check.% e2e-tests: ## Run the end-to-end tests against the service containers - docker run -t --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash -c 'paver e2e_test --exclude="whitelabel\|enterprise"' + docker-compose run --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash -c 'paver e2e_test --exclude="whitelabel\|enterprise"' validate-lms-volume: ## Validate that changes to the local workspace are reflected in the LMS container touch $(DEVSTACK_WORKSPACE)/edx-platform/testfile - docker exec edx.devstack.lms ls /edx/app/edxapp/edx-platform/testfile + docker-compose exec -T lms ls /edx/app/edxapp/edx-platform/testfile rm $(DEVSTACK_WORKSPACE)/edx-platform/testfile vnc-passwords: ## Get the VNC passwords for the Chrome and Firefox Selenium containers - @docker logs edx.devstack.chrome 2>&1 | grep "VNC password" | tail -1 - @docker logs edx.devstack.firefox 2>&1 | grep "VNC password" | tail -1 + @docker-compose logs chrome 2>&1 | grep "VNC password" | tail -1 + @docker-compose logs firefox 2>&1 | grep "VNC password" | tail -1 devpi-password: ## Get the root devpi password for the devpi container docker-compose exec devpi bash -c "cat /data/server/.serverpassword" @@ -393,7 +393,7 @@ dev.up.analytics_pipeline: dev.up.analyticspipeline ## Bring up analytics pipeli pull.analytics_pipeline: dev.pull.analyticspipeline ## Update analytics pipeline docker images analytics-pipeline-devstack-test: ## Run analytics pipeline tests in travis build - docker exec -u hadoop -i edx.devstack.analytics_pipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity' + docker-compose $(DOCKER_COMPOSE_FILES) exec -u hadoop -T analyticspipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity' stop.analytics_pipeline: ## Stop all Analytics pipeline services. docker-compose $(DOCKER_COMPOSE_FILES) stop \ @@ -401,7 +401,7 @@ stop.analytics_pipeline: ## Stop all Analytics pipeline services. sparkworker vertica analyticspipeline hadoop-application-logs-%: ## View hadoop logs by application Id - docker exec -it edx.devstack.analytics_pipeline.nodemanager yarn logs -applicationId $* + docker-compose exec nodemanager yarn logs -applicationId $* # Provisions studio, ecommerce, and marketing with course(s) in test-course.json # Modify test-course.json before running this make target to generate a custom course diff --git a/Makefile.edx b/Makefile.edx index 81343ba2de..ec0e0cc2a7 100644 --- a/Makefile.edx +++ b/Makefile.edx @@ -23,12 +23,12 @@ dev.provision.whitelabel: # The containers must be started with the 'dev.up.e2e_wl_tests' target. # AND the test must be setup using the 'dev.provision.whitelabel' target. whitelabel-tests: - docker run -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e - docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh devstack.whitelabel:/tmp/run_whitelabel_tests.sh - docker exec -t devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh + docker-compose run -T -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e + docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh $(docker-compose ps -q devstack.whitelabel)":/tmp/run_whitelabel_tests.sh + docker-compose exec devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh whitelabel-cleanup: - docker rm --force devstack.whitelabel + docker-compose rm --force devstack.whitelabel whitelabel-shell: ## Start a whitelabel test shell session - docker exec -it devstack.whitelabel env TERM=$(TERM) bash + docker-compose exec devstack.whitelabel env TERM=$(TERM) bash diff --git a/course-generator/create-courses.sh b/course-generator/create-courses.sh index eea96f6589..b7ee059a9e 100755 --- a/course-generator/create-courses.sh +++ b/course-generator/create-courses.sh @@ -15,14 +15,14 @@ for arg in "$@"; do studio=true fi elif [ $arg == "--ecommerce" ]; then - if [ ! "$(docker exec -t edx.devstack.ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then + if [ ! "$(docker-compose exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then echo "Issue with ecommerce container" container_error=true else ecommerce=true fi elif [ $arg == "--marketing" ]; then - if [ ! "$(docker exec -t edx.devstack.marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then + if [ ! "$(docker-compose exec marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then echo "Issue with marketing container. Course creation will proceed without marketing container." else marketing=true @@ -54,10 +54,10 @@ fi if $ecommerce ; then echo "Creating courses on ecommerce." - docker exec -t edx.devstack.ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" + docker-compose exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" fi if $marketing ; then echo "Creating courses on marketing." - docker exec -t edx.devstack.marketing bash -c "drush generate_courses '$course_json'" + docker-compose exec marketing bash -c "drush generate_courses '$course_json'" fi diff --git a/docker-compose-analytics-pipeline.yml b/docker-compose-analytics-pipeline.yml index aa84af84a6..f9df62df8e 100644 --- a/docker-compose-analytics-pipeline.yml +++ b/docker-compose-analytics-pipeline.yml @@ -3,10 +3,13 @@ version: "2.1" services: namenode: image: edxops/analytics_pipeline_hadoop_namenode:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.namenode hostname: namenode environment: - CLUSTER_NAME=devstack + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.namenode ports: - 127.0.0.1:50070:50070 command: ["/run.sh"] @@ -15,12 +18,15 @@ services: datanode: image: edxops/analytics_pipeline_hadoop_datanode:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.datanode hostname: datanode environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" depends_on: - namenode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.datanode ports: - 127.0.0.1:50075:50075 command: ["/run.sh"] @@ -29,7 +35,6 @@ services: resourcemanager: image: edxops/analytics_pipeline_hadoop_resourcemanager:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.resourcemanager hostname: resourcemanager environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" @@ -40,13 +45,16 @@ services: depends_on: - namenode - datanode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.resourcemanager ports: - 127.0.0.1:8088:8088 # resource manager web ui command: ["/run.sh"] nodemanager: image: edxops/analytics_pipeline_hadoop_nodemanager:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.nodemanager hostname: nodemanager environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" @@ -60,6 +68,10 @@ services: - resourcemanager - namenode - datanode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.nodemanager ports: - 127.0.0.1:8042:8042 # node manager web ui - 127.0.0.1:19888:19888 # node manager job history server ui @@ -67,8 +79,11 @@ services: sparkmaster: image: edxops/analytics_pipeline_spark_master:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.sparkmaster hostname: sparkmaster + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.sparkmaster ports: - 127.0.0.1:8080:8080 - 127.0.0.1:7077:7077 # spark master port @@ -77,25 +92,30 @@ services: sparkworker: image: edxops/analytics_pipeline_spark_worker:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.sparkworker hostname: sparkworker depends_on: - sparkmaster environment: - SPARK_MASTER=spark://sparkmaster:7077 + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.sparkworker ports: - 127.0.0.1:8081:8081 # spark worker UI vertica: image: iamamr/vertica:9.1.0-0 hostname: vertica - container_name: edx.devstack.analytics_pipeline.vertica + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.vertica volumes: - vertica_data:/home/dbadmin/docker analyticspipeline: image: edxops/analytics_pipeline:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline hostname: analyticspipeline volumes: - ${DEVSTACK_WORKSPACE}/edx-analytics-pipeline:/edx/app/analytics_pipeline/analytics_pipeline @@ -109,6 +129,10 @@ services: - sparkworker - elasticsearch - vertica + networks: + default: + aliases: + - edx.devstack.analytics_pipeline ports: - 127.0.0.1:4040:4040 # spark web UI environment: diff --git a/docker-compose-marketing-site.yml b/docker-compose-marketing-site.yml index 4bd2693b7a..2ae9269eba 100644 --- a/docker-compose-marketing-site.yml +++ b/docker-compose-marketing-site.yml @@ -7,7 +7,6 @@ services: - MARKETING_SITE_ROOT="http://localhost:8080" marketing: - container_name: edx.devstack.marketing depends_on: - mysql - memcached @@ -26,5 +25,9 @@ services: # IP address of your machine to enable debugging (IP_ADDRESS set in .env file) - XDEBUG_CONFIG=remote_host=${XDEBUG_IP_ADDRESS:-127.0.0.1} image: edxops/edx-mktg:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.marketing ports: - "8080:80" diff --git a/docker-compose-watchers-nfs.yml b/docker-compose-watchers-nfs.yml index 532a49060f..adfb8efc27 100644 --- a/docker-compose-watchers-nfs.yml +++ b/docker-compose-watchers-nfs.yml @@ -3,11 +3,14 @@ version: "2.1" services: lms_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.lms_watcher environment: BOK_CHOY_HOSTNAME: edx.devstack.lms_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.lms_watcher volumes: - edx-nfs:/edx/app/edxapp/edx-platform - edxapp_lms_assets:/edx/var/edxapp/staticfiles/ @@ -17,11 +20,14 @@ services: studio_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.studio_watcher environment: BOK_CHOY_HOSTNAME: edx.devstack.studio_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.studio_watcher volumes: - edxapp_studio_assets:/edx/var/edxapp/staticfiles/ - edx-nfs:/edx/app/edxapp/edx-platform diff --git a/docker-compose-watchers.yml b/docker-compose-watchers.yml index 7c7828f701..ddf6ab98fa 100644 --- a/docker-compose-watchers.yml +++ b/docker-compose-watchers.yml @@ -3,11 +3,14 @@ version: "2.1" services: lms_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.lms_watcher environment: BOK_CHOY_HOSTNAME: edx.devstack.lms_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.lms_watcher volumes: - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform:cached - edxapp_lms_assets:/edx/var/edxapp/staticfiles/ @@ -17,11 +20,14 @@ services: studio_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.studio_watcher environment: BOK_CHOY_HOSTNAME: edx.devstack.studio_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.studio_watcher volumes: - edxapp_studio_assets:/edx/var/edxapp/staticfiles/ - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform:cached diff --git a/docker-compose-xqueue.yml b/docker-compose-xqueue.yml index bf4a73ac54..8893313711 100644 --- a/docker-compose-xqueue.yml +++ b/docker-compose-xqueue.yml @@ -2,19 +2,25 @@ version: "2.1" services: xqueue: - container_name: edx.devstack.xqueue image: edxops/xqueue:${OPENEDX_RELEASE:-latest} command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py runserver 0.0.0.0:18040 ; sleep 2; done' volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file + networks: + default: + aliases: + - edx.devstack.xqueue ports: - 18040:18040 xqueue_consumer: - container_name: edx.devstack.xqueue_consumer image: edxops/xqueue:${OPENEDX_RELEASE:-latest} command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py run_consumer ; sleep 2; done' + networks: + default: + aliases: + - edx.devstack.xqueue_consumer volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file diff --git a/docker-compose.yml b/docker-compose.yml index d96017260e..bd5fb94f03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,10 +17,13 @@ services: # ================================================ chrome: - container_name: edx.devstack.chrome hostname: chrome.devstack.edx image: edxops/chrome:${OPENEDX_RELEASE:-latest} shm_size: 2g + networks: + default: + aliases: + - edx.devstack.chrome ports: - "15900:5900" volumes: # for file uploads @@ -28,18 +31,24 @@ services: - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data devpi: - container_name: edx.devstack.devpi hostname: devpi.devstack.edx image: edxops/devpi:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.devpi ports: - "3141:3141" volumes: - devpi_data:/data elasticsearch: - container_name: edx.devstack.elasticsearch hostname: elasticsearch.devstack.edx image: edxops/elasticsearch:devstack + networks: + default: + aliases: + - edx.devstack.elasticsearch # TODO: What to do about these forwarded ports? They'll conflict with ports forwarded by the Vagrant VM. # ports: # - "9200:9200" @@ -50,15 +59,21 @@ services: # This is meant to be used to test ES upgrades so that we do not have to upgrade all of our services to ES5 at once. elasticsearch-5: - container_name: edx.devstack.elasticsearch-5 hostname: elasticsearch-5.devstack.edx image: elasticsearch:5.6.16 + networks: + default: + aliases: + - edx.devstack.elasticsearch-5 firefox: - container_name: edx.devstack.firefox hostname: firefox.devstack.edx image: edxops/firefox:${OPENEDX_RELEASE:-latest} shm_size: 2g + networks: + default: + aliases: + - edx.devstack.firefox ports: - "25900:5900" volumes: # for file uploads @@ -66,9 +81,12 @@ services: - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data memcached: - container_name: edx.devstack.memcached hostname: memcached.devstack.edx image: memcached:1.5.10-alpine + networks: + default: + aliases: + - edx.devstack.memcached # ports: # - "11211:11211" @@ -77,9 +95,12 @@ services: # to conserve disk space, and disable the journal for a minor performance gain. # See https://docs.mongodb.com/v3.0/reference/program/mongod/#options for complete details. command: mongod --smallfiles --nojournal --storageEngine wiredTiger - container_name: edx.devstack.mongo hostname: mongo.devstack.edx image: mongo:${MONGO_VERSION:-3.6.17} + networks: + default: + aliases: + - edx.devstack.mongo # ports: # - "27017:27017" volumes: @@ -87,35 +108,44 @@ services: mysql: command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci - container_name: edx.devstack.mysql hostname: mysql.devstack.edx environment: MYSQL_ROOT_PASSWORD: "" MYSQL_ALLOW_EMPTY_PASSWORD: "yes" image: mysql:5.6 - # ports: - # - "3506:3306" + networks: + default: + aliases: + - edx.devstack.mysql + ports: + - "3506:3306" volumes: - mysql_data:/var/lib/mysql mysql57: command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci - container_name: edx.devstack.mysql57 hostname: mysql57.devstack.edx environment: MYSQL_ROOT_PASSWORD: "" MYSQL_ALLOW_EMPTY_PASSWORD: "yes" image: mysql:5.7 + networks: + default: + aliases: + - edx.devstack.mysql57 # ports: # - "3506:3306" volumes: - mysql57_data:/var/lib/mysql redis: - container_name: edx.devstack.redis hostname: redis.devstack.edx image: redis:2.8 command: redis-server --requirepass password + networks: + default: + aliases: + - edx.devstack.redis # ================================================ # edX services @@ -123,7 +153,6 @@ services: credentials: command: bash -c 'source /edx/app/credentials/credentials_env && while true; do python /edx/app/credentials/credentials/manage.py runserver 0.0.0.0:18150; sleep 2; done' - container_name: edx.devstack.credentials hostname: credentials.devstack.edx depends_on: - mysql @@ -139,12 +168,15 @@ services: ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/credentials:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.credentials ports: - "18150:18150" discovery: command: bash -c 'source /edx/app/discovery/discovery_env && while true; do python /edx/app/discovery/discovery/manage.py runserver 0.0.0.0:18381; sleep 2; done' - container_name: edx.devstack.discovery hostname: discovery.devstack.edx depends_on: - mysql @@ -158,6 +190,10 @@ services: ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/discovery:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.discovery ports: - "18381:18381" volumes: @@ -165,7 +201,6 @@ services: ecommerce: command: bash -c 'source /edx/app/ecommerce/ecommerce_env && while true; do python /edx/app/ecommerce/ecommerce/manage.py runserver 0.0.0.0:18130; sleep 2; done' - container_name: edx.devstack.ecommerce hostname: ecommerce.devstack.edx depends_on: - mysql @@ -179,18 +214,25 @@ services: DJANGO_WATCHMAN_TIMEOUT: 30 ENABLE_DJANGO_TOOLBAR: 1 image: edxops/ecommerce:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.ecommerce ports: - "18130:18130" edx_notes_api: command: bash -c 'source /edx/app/edx_notes_api/edx_notes_api_env && while true; do python /edx/app/edx_notes_api/edx_notes_api/manage.py runserver 0.0.0.0:18120 --settings notesserver.settings.devstack; sleep 2; done' - container_name: edx.devstack.edxnotesapi hostname: edx_notes_api.devstack.edx depends_on: - devpi - elasticsearch - mysql image: edxops/notes:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.edxnotesapi ports: - "18120:18120" environment: @@ -206,19 +248,21 @@ services: forum: command: bash -c 'source /edx/app/forum/ruby_env && source /edx/app/forum/devstack_forum_env && cd /edx/app/forum/cs_comments_service && bundle install && while true; do ruby app.rb -o 0.0.0.0 ; sleep 2; done' - container_name: edx.devstack.forum hostname: forum.devstack.edx depends_on: - mongo - memcached - elasticsearch image: edxops/forum:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.forum ports: - "44567:4567" lms: command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack_docker; sleep 2; done' - container_name: edx.devstack.lms hostname: lms.devstack.edx depends_on: - devpi @@ -240,6 +284,10 @@ services: NO_PYTHON_UNINSTALL: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.lms ports: - "18000:18000" - "19876:19876" # JS test debugging @@ -250,7 +298,6 @@ services: registrar: command: bash -c 'source /edx/app/registrar/registrar_env && while true; do python /edx/app/registrar/registrar/manage.py runserver 0.0.0.0:18734; sleep 2; done' - container_name: edx.devstack.registrar hostname: registrar.devstack.edx depends_on: - discovery @@ -278,6 +325,10 @@ services: CELERY_BROKER_PASSWORD: password DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/registrar:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.registrar ports: - "18734:18734" volumes: @@ -285,7 +336,6 @@ services: registrar-worker: command: bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && celery -A registrar worker -l debug -c 2' - container_name: edx.devstack.registrar-worker hostname: registrar-worker.devstack.edx depends_on: - lms @@ -308,6 +358,10 @@ services: CELERY_BROKER_PASSWORD: password DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/registrar:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.registrar-worker ports: - "18735:18735" volumes: @@ -315,7 +369,6 @@ services: studio: command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py cms runserver 0.0.0.0:18010 --settings devstack_docker; sleep 2; done' - container_name: edx.devstack.studio hostname: studio.devstack.edx depends_on: - devpi @@ -336,6 +389,10 @@ services: NO_PYTHON_UNINSTALL: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.studio ports: - "18010:18010" - "19877:19877" # JS test debugging @@ -357,7 +414,10 @@ services: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/frontend-app-publisher' - container_name: edx.devstack.frontend-app-publisher + networks: + default: + aliases: + - edx.devstack.frontend-app-publisher ports: - "18400:18400" depends_on: @@ -368,7 +428,10 @@ services: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/gradebook' - container_name: edx.devstack.gradebook + networks: + default: + aliases: + - edx.devstack.gradebook ports: - "1994:1994" depends_on: @@ -379,7 +442,10 @@ services: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/program-console' - container_name: edx.devstack.program-console + networks: + default: + aliases: + - edx.devstack.program-console ports: - "1976:1976" depends_on: diff --git a/gather-feature-toggle-state.sh b/gather-feature-toggle-state.sh index 2ed9946195..5c9bce2272 100644 --- a/gather-feature-toggle-state.sh +++ b/gather-feature-toggle-state.sh @@ -12,20 +12,20 @@ if [ -e feature-toggle-data ]; then fi mkdir feature-toggle-data -docker exec -t edx.devstack.credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py dumpdata waffle --format=json > /edx/app/credentials/credentials/credentials_waffle.json' -docker cp edx.devstack.credentials:/edx/app/credentials/credentials/credentials_waffle.json feature-toggle-data -docker exec -t edx.devstack.credentials bash -c 'rm /edx/app/credentials/credentials/credentials_waffle.json' +docker-compose exec credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py dumpdata waffle --format=json > /edx/app/credentials/credentials/credentials_waffle.json' +docker cp "$(docker-compose ps -q credentials)":/edx/app/credentials/credentials/credentials_waffle.json feature-toggle-data +docker-compose exec credentials bash -c 'rm /edx/app/credentials/credentials/credentials_waffle.json' -docker exec -t edx.devstack.discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py dumpdata waffle --format=json > /edx/app/discovery/discovery/discovery_waffle.json' -docker cp edx.devstack.discovery:/edx/app/discovery/discovery/discovery_waffle.json feature-toggle-data -docker exec -t edx.devstack.discovery bash -c '> /edx/app/discovery/discovery/discovery_waffle.json' +docker-compose exec discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py dumpdata waffle --format=json > /edx/app/discovery/discovery/discovery_waffle.json' +docker cp "$(docker-compose ps -q discovery)":/edx/app/discovery/discovery/discovery_waffle.json feature-toggle-data +docker-compose exec discovery bash -c '> /edx/app/discovery/discovery/discovery_waffle.json' -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py dumpdata waffle --format=json > /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' -docker cp edx.devstack.ecommerce:/edx/app/ecommerce/ecommerce/ecommerce_waffle.json feature-toggle-data -docker exec -t edx.devstack.ecommerce bash -c 'rm /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' +docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py dumpdata waffle --format=json > /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' +docker cp "$(docker-compose ps -q ecommerce)":/edx/app/ecommerce/ecommerce/ecommerce_waffle.json feature-toggle-data +docker-compose exec ecommerce bash -c 'rm /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle --format=json > /edx/app/edxapp/edx-platform/lms_waffle.json' -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle_utils --format=json > /edx/app/edxapp/edx-platform/lms_waffle_utils.json' -docker cp edx.devstack.lms:/edx/app/edxapp/edx-platform/lms_waffle.json feature-toggle-data -docker cp edx.devstack.lms:/edx/app/edxapp/edx-platform/lms_waffle_utils.json feature-toggle-data -docker exec -t edx.devstack.lms bash -c 'rm /edx/app/edxapp/edx-platform/lms_waffle*.json' +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle --format=json > /edx/app/edxapp/edx-platform/lms_waffle.json' +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle_utils --format=json > /edx/app/edxapp/edx-platform/lms_waffle_utils.json' +docker cp "$(docker-compose ps -q lms)":/edx/app/edxapp/edx-platform/lms_waffle.json feature-toggle-data +docker cp "$(docker-compose ps -q lms)":/edx/app/edxapp/edx-platform/lms_waffle_utils.json feature-toggle-data +docker-compose exec lms bash -c 'rm /edx/app/edxapp/edx-platform/lms_waffle*.json' diff --git a/load-db.sh b/load-db.sh index 24b70fada5..912c091aa9 100755 --- a/load-db.sh +++ b/load-db.sh @@ -17,5 +17,6 @@ then fi echo "Loading the $1 database..." -docker exec -i edx.devstack.mysql mysql -uroot $1 < $1.sql +#docker exec -i edx.devstack.mysql mysql -uroot $1 < $1.sql +docker-compose exec -T mysql bash -c "mysql -uroot $1" < $1.sql echo "Finished loading the $1 database!" diff --git a/marketing.mk b/marketing.mk index 16a8adb4a2..c1496c3ef0 100644 --- a/marketing.mk +++ b/marketing.mk @@ -3,7 +3,7 @@ help-marketing: ## Display this help message @awk -F ':.*?## ' '/^[a-zA-Z]/ && NF==2 {printf "\033[36m %-25s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | grep marketing | sort marketing-shell: ## Run a shell on the marketing site container - docker exec -it edx.devstack.marketing env TERM=$(TERM) bash -c 'cd /edx/app/edx-mktg/edx-mktg; exec /bin/bash -sh' + docker-compose exec marketing env TERM=$(TERM) bash -c 'cd /edx/app/edx-mktg/edx-mktg; exec /bin/bash -sh' stop-marketing: ## Stop all services (including the marketing site) with host volumes docker-compose -f docker-compose.yml -f docker-compose-host.yml -f docker-compose-themes.yml -f docker-compose-marketing-site.yml -f docker-compose-marketing-site-host.yml stop diff --git a/options.mk b/options.mk index 5daa0ac414..856e9f7494 100644 --- a/options.mk +++ b/options.mk @@ -10,8 +10,12 @@ DEVSTACK_WORKSPACE ?= $(shell pwd)/.. # Name of Docker Compose project. # See https://docs.docker.com/compose/reference/envvars/#compose_project_name -# Defaults to 'devstack'. -COMPOSE_PROJECT_NAME ?= devstack +# Defaults to 'devstack-master' should OPENEDX_RELEASE not be defined. +ifdef OPENEDX_RELEASE + COMPOSE_PROJECT_NAME ?= devstack-${OPENEDX_RELEASE} +else + COMPOSE_PROJECT_NAME ?= devstack-master +endif # increase Docker Compose HTTP timeout so that devstack provisioning does not fail in unstable networks COMPOSE_HTTP_TIMEOUT=180 diff --git a/provision-analyticspipeline.sh b/provision-analyticspipeline.sh index 70e458e683..bb09d437d8 100755 --- a/provision-analyticspipeline.sh +++ b/provision-analyticspipeline.sh @@ -22,12 +22,12 @@ docker-compose $DOCKER_COMPOSE_FILES up -d analyticspipeline # Analytics pipeline has dependency on lms but we only need its db schema & not full lms. So we'll just load their db # schemas as part of analytics pipeline provisioning. If there is a need of a fully fledged LMS, then provision lms # by following their documentation. -if [[ ! -z "`docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='edxapp'" 2>&1`" ]]; +if [[ ! -z "`docker-compose exec mysql bash -c "mysql -uroot -se \"SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='edxapp'\"" 2>&1`" ]]; then echo -e "${GREEN}LMS DB exists, skipping lms schema load.${NC}" else echo -e "${GREEN}LMS DB not found, provisioning lms schema.${NC}" - docker exec -i edx.devstack.mysql mysql -uroot mysql < provision.sql + docker-compose exec -T mysql bash -c 'mysql -uroot mysql' < provision.sql ./load-db.sh edxapp docker-compose $DOCKER_COMPOSE_FILES up -d lms studio docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' @@ -39,7 +39,7 @@ fi echo -e "${GREEN}LMS database provisioned successfully...${NC}" echo -e "${GREEN}Creating databases and users...${NC}" -docker exec -i edx.devstack.mysql mysql -uroot mysql < provision-analytics-pipeline.sql +docker-compose exec -T mysql bash -c 'mysql -uroot mysql' < provision-analytics-pipeline.sql # initialize hive metastore echo -e "${GREEN}Initializing HIVE metastore...${NC}" @@ -56,4 +56,4 @@ done sleep 10 # for datanode & other services to activate echo -e "${GREEN}Namenode is ready!${NC}" -docker exec -u hadoop -i edx.devstack.analytics_pipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' +docker-compose $DOCKER_COMPOSE_FILES exec -u hadoop analyticspipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' diff --git a/provision-credentials.sh b/provision-credentials.sh index 32fccdaf61..0334f2966b 100755 --- a/provision-credentials.sh +++ b/provision-credentials.sh @@ -9,20 +9,20 @@ port=18150 docker-compose $DOCKER_COMPOSE_FILES up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" echo -e "${GREEN}Configuring site for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' +docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker-compose exec ${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-e2e.sh b/provision-e2e.sh index b9e7b5508e..b245bd0d22 100755 --- a/provision-e2e.sh +++ b/provision-e2e.sh @@ -10,7 +10,7 @@ elif [ ! -d "$DEVSTACK_WORKSPACE" ]; then fi # Copy the test course tarball into the studio container -docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz edx.devstack.studio:/tmp/ +docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz "$(docker-compose ps -q studio)":/tmp/ # Extract the test course tarball docker-compose exec -T studio bash -c 'cd /tmp && tar xzf course.tar.gz' diff --git a/provision-ecommerce.sh b/provision-ecommerce.sh index c1680fab02..75353fcd6f 100755 --- a/provision-ecommerce.sh +++ b/provision-ecommerce.sh @@ -4,6 +4,6 @@ ./provision-ida.sh ecommerce ecommerce 18130 # Configure ecommerce -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/' -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' +docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/' +docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' +docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' diff --git a/provision-ida-user.sh b/provision-ida-user.sh index 1a3b93c9ee..965de8a566 100755 --- a/provision-ida-user.sh +++ b/provision-ida-user.sh @@ -7,8 +7,8 @@ client_port=$3 echo -e "${GREEN}Creating service user and OAuth2 applications for ${app_name}...${NC}" # Create the service user. -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" # Create the DOT applications - one for single sign-on and one for backend service IDA-to-IDA authentication. -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "http://localhost:$3/complete/edx-oauth2/" --client-id "$1-sso-key" --client-secret "$1-sso-secret" --scopes "user_id" $1-sso $1_worker' -- "$app_name" "$client_name" "$client_port" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "$1-backend-service-key" --client-secret "$1-backend-service-secret" $1-backend-service $1_worker' -- "$app_name" "$client_name" "$client_port" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "http://localhost:$3/complete/edx-oauth2/" --client-id "$1-sso-key" --client-secret "$1-sso-secret" --scopes "user_id" $1-sso $1_worker' -- "$app_name" "$client_name" "$client_port" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "$1-backend-service-key" --client-secret "$1-backend-service-secret" $1-backend-service $1_worker' -- "$app_name" "$client_name" "$client_port" diff --git a/provision-ida.sh b/provision-ida.sh index 62bbf4d0ad..9ed579ffb5 100755 --- a/provision-ida.sh +++ b/provision-ida.sh @@ -7,17 +7,17 @@ container_name=${4:-$1} # (Optional) The name of the container. If missing, wil docker-compose $DOCKER_COMPOSE_FILES up -d $app_name echo -e "${GREEN}Installing requirements for ${app_name}...${NC}" -docker exec -t edx.devstack.${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" +docker-compose exec ${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" echo -e "${GREEN}Running migrations for ${app_name}...${NC}" -docker exec -t edx.devstack.${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" +docker-compose exec ${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" echo -e "${GREEN}Creating super-user for ${app_name}...${NC}" -docker exec -t edx.devstack.${container_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" +docker-compose exec ${container_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" ./provision-ida-user.sh $app_name $client_name $client_port # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${app_name}...${NC}" -docker exec -t edx.devstack.${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" +docker-compose exec ${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" diff --git a/provision-notes.sh b/provision-notes.sh index a840408741..12ff3767fb 100755 --- a/provision-notes.sh +++ b/provision-notes.sh @@ -1,8 +1,8 @@ # Provisioning script for the notes service # Common provisioning tasks for IDAs, including requirements, migrations, oauth client creation, etc. -./provision-ida.sh edx_notes_api edx-notes 18120 edxnotesapi +./provision-ida.sh edx_notes_api edx-notes 18120 edx_notes_api # This will build the elasticsearch index for notes. echo -e "${GREEN}Creating indexes for edx_notes_api...${NC}" -docker exec -t edx.devstack.edxnotesapi bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py rebuild_index --noinput' -- edx_notes_api +docker-compose exec edx_notes_api bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py rebuild_index --noinput' -- edx_notes_api diff --git a/provision-registrar.sh b/provision-registrar.sh index 3fd78f683c..c96bfb8843 100755 --- a/provision-registrar.sh +++ b/provision-registrar.sh @@ -6,17 +6,17 @@ port=18734 docker-compose $DOCKER_COMPOSE_FILES up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make requirements' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make migrate' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make createsuperuser' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make createsuperuser' -- "$name" ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c ' if ! source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make static 2>registrar_make_static.err; then echo "------- Last 100 lines of stderr"; tail regsitrar_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker-compose exec ${name} bash -c ' if ! source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make static 2>registrar_make_static.err; then echo "------- Last 100 lines of stderr"; tail regsitrar_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-retirement-user.sh b/provision-retirement-user.sh index b3a5dac2e3..3267258628 100755 --- a/provision-retirement-user.sh +++ b/provision-retirement-user.sh @@ -4,5 +4,5 @@ app_name=$1 user_name=$2 echo -e "${GREEN}Creating retirement service user ${user_name} and DOT Application ${app_name}...${NC}" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" diff --git a/provision.sh b/provision.sh index 60e6e04bfe..42761b93c3 100755 --- a/provision.sh +++ b/provision.sh @@ -122,7 +122,7 @@ fi # Ensure the MySQL server is online and usable echo "${GREEN}Waiting for MySQL.${NC}" -until docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')" &> /dev/null +until docker-compose exec -T mysql bash -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 @@ -136,20 +136,20 @@ echo -e "${GREEN}MySQL ready.${NC}" # Ensure that the MySQL databases and users are created for all IDAs. # (A no-op for databases and users that already exist). echo -e "${GREEN}Ensuring MySQL databases and users exist...${NC}" -docker exec -i edx.devstack.mysql mysql -uroot mysql < provision.sql +docker-compose exec -T mysql bash -c "mysql -uroot mysql" < provision.sql # If necessary, ensure the MongoDB server is online and usable # and create its users. if needs_mongo "$to_provision"; then echo -e "${GREEN}Waiting for MongoDB...${NC}" - until docker exec -i edx.devstack.mongo mongo --eval "printjson(db.serverStatus())" &> /dev/null + until docker-compose exec -T mongo bash -c 'mongo --eval "printjson(db.serverStatus())"' &> /dev/null do printf "." sleep 1 done echo -e "${GREEN}MongoDB ready.${NC}" echo -e "${GREEN}Creating MongoDB users...${NC}" - docker exec -i edx.devstack.mongo mongo < mongo-provision.js + docker-compose exec -T mongo bash -c "mongo" < mongo-provision.js else echo -e "${GREEN}MongoDB preparation not required; skipping.${NC}" fi diff --git a/upgrade_mongo_3_6.sh b/upgrade_mongo_3_6.sh index 1d0ce7c9c2..e5a0ac4c13 100755 --- a/upgrade_mongo_3_6.sh +++ b/upgrade_mongo_3_6.sh @@ -13,7 +13,7 @@ echo -e "${GREEN}Sarting Mongo ${MONGO_VERSION}${NC}" docker-compose up -d mongo echo -e "${GREEN}Waiting for MongoDB...${NC}" -until docker exec -i edx.devstack.mongo mongo --eval "printjson(db.serverStatus())" &> /dev/null +until docker-compose exec -T mongo bash -c 'mongo --eval \"printjson(db.serverStatus())\"' &> /dev/null do if docker-compose logs mongo | grep -q "BadValue: Invalid value for version, found 3.6, expected '3.4' or '3.2'"; then echo -e "${YELLOW}Already upgraded to Mongo 3.6, exiting${NC}" @@ -45,7 +45,7 @@ echo -e "${GREEN}Restarting Mongo on version ${MONGO_VERSION}${NC}" docker-compose up -d mongo echo -e "${GREEN}Waiting for MongoDB...${NC}" -until docker exec -i edx.devstack.mongo mongo --eval "printjson(db.serverStatus())" &> /dev/null +until docker-compose exec -T mongo bash -c 'mongo --eval \"printjson(db.serverStatus())\"' &> /dev/null do printf "." sleep 1 From 61464293f1e8ea9cca90d28770ef56efc0d1aeb9 Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Fri, 8 May 2020 16:06:53 -0400 Subject: [PATCH 02/11] Fix additional changes recommended by edX. --- load-db.sh | 1 - options.mk | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/load-db.sh b/load-db.sh index 912c091aa9..7ca21dae64 100755 --- a/load-db.sh +++ b/load-db.sh @@ -17,6 +17,5 @@ then fi echo "Loading the $1 database..." -#docker exec -i edx.devstack.mysql mysql -uroot $1 < $1.sql docker-compose exec -T mysql bash -c "mysql -uroot $1" < $1.sql echo "Finished loading the $1 database!" diff --git a/options.mk b/options.mk index 856e9f7494..1c63c0d1bd 100644 --- a/options.mk +++ b/options.mk @@ -10,11 +10,11 @@ DEVSTACK_WORKSPACE ?= $(shell pwd)/.. # Name of Docker Compose project. # See https://docs.docker.com/compose/reference/envvars/#compose_project_name -# Defaults to 'devstack-master' should OPENEDX_RELEASE not be defined. +# Defaults to 'devstack' should OPENEDX_RELEASE not be defined. ifdef OPENEDX_RELEASE COMPOSE_PROJECT_NAME ?= devstack-${OPENEDX_RELEASE} else - COMPOSE_PROJECT_NAME ?= devstack-master + COMPOSE_PROJECT_NAME ?= devstack endif # increase Docker Compose HTTP timeout so that devstack provisioning does not fail in unstable networks From df73f42bdb4789e86cc62f4d3400475a209f634c Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Mon, 18 May 2020 18:20:28 -0400 Subject: [PATCH 03/11] Added back in `container_names` to avoid breaking other developers code with `docker` command calls. This makes container names more easily predictable by external services and for backward compatibility. Thanks @arbrandes and @kdmccormick for the recommended changes. --- docker-compose-analytics-pipeline.yml | 40 ++------- docker-compose-marketing-site.yml | 5 +- docker-compose-watchers-nfs.yml | 10 +-- docker-compose-watchers.yml | 10 +-- docker-compose-xqueue.yml | 10 +-- docker-compose.yml | 114 ++++++-------------------- 6 files changed, 39 insertions(+), 150 deletions(-) diff --git a/docker-compose-analytics-pipeline.yml b/docker-compose-analytics-pipeline.yml index f9df62df8e..f4af248325 100644 --- a/docker-compose-analytics-pipeline.yml +++ b/docker-compose-analytics-pipeline.yml @@ -3,13 +3,10 @@ version: "2.1" services: namenode: image: edxops/analytics_pipeline_hadoop_namenode:${OPENEDX_RELEASE:-latest} + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.namenode" hostname: namenode environment: - CLUSTER_NAME=devstack - networks: - default: - aliases: - - edx.devstack.analytics_pipeline.namenode ports: - 127.0.0.1:50070:50070 command: ["/run.sh"] @@ -18,15 +15,12 @@ services: datanode: image: edxops/analytics_pipeline_hadoop_datanode:${OPENEDX_RELEASE:-latest} + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.datanode" hostname: datanode environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" depends_on: - namenode - networks: - default: - aliases: - - edx.devstack.analytics_pipeline.datanode ports: - 127.0.0.1:50075:50075 command: ["/run.sh"] @@ -35,6 +29,7 @@ services: resourcemanager: image: edxops/analytics_pipeline_hadoop_resourcemanager:${OPENEDX_RELEASE:-latest} + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.resourcemanager" hostname: resourcemanager environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" @@ -45,16 +40,13 @@ services: depends_on: - namenode - datanode - networks: - default: - aliases: - - edx.devstack.analytics_pipeline.resourcemanager ports: - 127.0.0.1:8088:8088 # resource manager web ui command: ["/run.sh"] nodemanager: image: edxops/analytics_pipeline_hadoop_nodemanager:${OPENEDX_RELEASE:-latest} + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.nodemanager" hostname: nodemanager environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" @@ -68,10 +60,6 @@ services: - resourcemanager - namenode - datanode - networks: - default: - aliases: - - edx.devstack.analytics_pipeline.nodemanager ports: - 127.0.0.1:8042:8042 # node manager web ui - 127.0.0.1:19888:19888 # node manager job history server ui @@ -79,11 +67,8 @@ services: sparkmaster: image: edxops/analytics_pipeline_spark_master:${OPENEDX_RELEASE:-latest} + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkmaster" hostname: sparkmaster - networks: - default: - aliases: - - edx.devstack.analytics_pipeline.sparkmaster ports: - 127.0.0.1:8080:8080 - 127.0.0.1:7077:7077 # spark master port @@ -92,30 +77,25 @@ services: sparkworker: image: edxops/analytics_pipeline_spark_worker:${OPENEDX_RELEASE:-latest} + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkworker" hostname: sparkworker depends_on: - sparkmaster environment: - SPARK_MASTER=spark://sparkmaster:7077 - networks: - default: - aliases: - - edx.devstack.analytics_pipeline.sparkworker ports: - 127.0.0.1:8081:8081 # spark worker UI vertica: image: iamamr/vertica:9.1.0-0 hostname: vertica - networks: - default: - aliases: - - edx.devstack.analytics_pipeline.vertica + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.vertica" volumes: - vertica_data:/home/dbadmin/docker analyticspipeline: image: edxops/analytics_pipeline:${OPENEDX_RELEASE:-latest} + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline" hostname: analyticspipeline volumes: - ${DEVSTACK_WORKSPACE}/edx-analytics-pipeline:/edx/app/analytics_pipeline/analytics_pipeline @@ -129,10 +109,6 @@ services: - sparkworker - elasticsearch - vertica - networks: - default: - aliases: - - edx.devstack.analytics_pipeline ports: - 127.0.0.1:4040:4040 # spark web UI environment: diff --git a/docker-compose-marketing-site.yml b/docker-compose-marketing-site.yml index 2ae9269eba..7ff39a2d0d 100644 --- a/docker-compose-marketing-site.yml +++ b/docker-compose-marketing-site.yml @@ -7,6 +7,7 @@ services: - MARKETING_SITE_ROOT="http://localhost:8080" marketing: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.marketing" depends_on: - mysql - memcached @@ -25,9 +26,5 @@ services: # IP address of your machine to enable debugging (IP_ADDRESS set in .env file) - XDEBUG_CONFIG=remote_host=${XDEBUG_IP_ADDRESS:-127.0.0.1} image: edxops/edx-mktg:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.marketing ports: - "8080:80" diff --git a/docker-compose-watchers-nfs.yml b/docker-compose-watchers-nfs.yml index adfb8efc27..4998189d31 100644 --- a/docker-compose-watchers-nfs.yml +++ b/docker-compose-watchers-nfs.yml @@ -3,14 +3,11 @@ version: "2.1" services: lms_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.lms_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.lms_watcher volumes: - edx-nfs:/edx/app/edxapp/edx-platform - edxapp_lms_assets:/edx/var/edxapp/staticfiles/ @@ -20,14 +17,11 @@ services: studio_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.studio_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.studio_watcher volumes: - edxapp_studio_assets:/edx/var/edxapp/staticfiles/ - edx-nfs:/edx/app/edxapp/edx-platform diff --git a/docker-compose-watchers.yml b/docker-compose-watchers.yml index ddf6ab98fa..1746fb2a83 100644 --- a/docker-compose-watchers.yml +++ b/docker-compose-watchers.yml @@ -3,14 +3,11 @@ version: "2.1" services: lms_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.lms_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.lms_watcher volumes: - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform:cached - edxapp_lms_assets:/edx/var/edxapp/staticfiles/ @@ -20,14 +17,11 @@ services: studio_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.studio_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.studio_watcher volumes: - edxapp_studio_assets:/edx/var/edxapp/staticfiles/ - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform:cached diff --git a/docker-compose-xqueue.yml b/docker-compose-xqueue.yml index 8893313711..c4f7bf006f 100644 --- a/docker-compose-xqueue.yml +++ b/docker-compose-xqueue.yml @@ -2,25 +2,19 @@ version: "2.1" services: xqueue: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue" image: edxops/xqueue:${OPENEDX_RELEASE:-latest} command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py runserver 0.0.0.0:18040 ; sleep 2; done' volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file - networks: - default: - aliases: - - edx.devstack.xqueue ports: - 18040:18040 xqueue_consumer: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue_consumer" image: edxops/xqueue:${OPENEDX_RELEASE:-latest} command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py run_consumer ; sleep 2; done' - networks: - default: - aliases: - - edx.devstack.xqueue_consumer volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file diff --git a/docker-compose.yml b/docker-compose.yml index bd5fb94f03..f997ba854e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,13 +17,10 @@ services: # ================================================ chrome: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.chrome" hostname: chrome.devstack.edx image: edxops/chrome:${OPENEDX_RELEASE:-latest} shm_size: 2g - networks: - default: - aliases: - - edx.devstack.chrome ports: - "15900:5900" volumes: # for file uploads @@ -31,24 +28,18 @@ services: - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data devpi: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.devpi" hostname: devpi.devstack.edx image: edxops/devpi:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.devpi ports: - "3141:3141" volumes: - devpi_data:/data elasticsearch: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.elasticsearch" hostname: elasticsearch.devstack.edx image: edxops/elasticsearch:devstack - networks: - default: - aliases: - - edx.devstack.elasticsearch # TODO: What to do about these forwarded ports? They'll conflict with ports forwarded by the Vagrant VM. # ports: # - "9200:9200" @@ -59,21 +50,15 @@ services: # This is meant to be used to test ES upgrades so that we do not have to upgrade all of our services to ES5 at once. elasticsearch-5: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.elasticsearch-5" hostname: elasticsearch-5.devstack.edx image: elasticsearch:5.6.16 - networks: - default: - aliases: - - edx.devstack.elasticsearch-5 firefox: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.firefox" hostname: firefox.devstack.edx image: edxops/firefox:${OPENEDX_RELEASE:-latest} shm_size: 2g - networks: - default: - aliases: - - edx.devstack.firefox ports: - "25900:5900" volumes: # for file uploads @@ -81,12 +66,9 @@ services: - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data memcached: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.memcached" hostname: memcached.devstack.edx image: memcached:1.5.10-alpine - networks: - default: - aliases: - - edx.devstack.memcached # ports: # - "11211:11211" @@ -95,12 +77,9 @@ services: # to conserve disk space, and disable the journal for a minor performance gain. # See https://docs.mongodb.com/v3.0/reference/program/mongod/#options for complete details. command: mongod --smallfiles --nojournal --storageEngine wiredTiger + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mongo" hostname: mongo.devstack.edx image: mongo:${MONGO_VERSION:-3.6.17} - networks: - default: - aliases: - - edx.devstack.mongo # ports: # - "27017:27017" volumes: @@ -108,44 +87,35 @@ services: mysql: command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql" hostname: mysql.devstack.edx environment: MYSQL_ROOT_PASSWORD: "" MYSQL_ALLOW_EMPTY_PASSWORD: "yes" image: mysql:5.6 - networks: - default: - aliases: - - edx.devstack.mysql - ports: - - "3506:3306" + # ports: + # - "3506:3306" volumes: - mysql_data:/var/lib/mysql mysql57: command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql57" hostname: mysql57.devstack.edx environment: MYSQL_ROOT_PASSWORD: "" MYSQL_ALLOW_EMPTY_PASSWORD: "yes" image: mysql:5.7 - networks: - default: - aliases: - - edx.devstack.mysql57 # ports: # - "3506:3306" volumes: - mysql57_data:/var/lib/mysql redis: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.redis" hostname: redis.devstack.edx image: redis:2.8 command: redis-server --requirepass password - networks: - default: - aliases: - - edx.devstack.redis # ================================================ # edX services @@ -153,6 +123,7 @@ services: credentials: command: bash -c 'source /edx/app/credentials/credentials_env && while true; do python /edx/app/credentials/credentials/manage.py runserver 0.0.0.0:18150; sleep 2; done' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.credentials" hostname: credentials.devstack.edx depends_on: - mysql @@ -168,15 +139,12 @@ services: ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/credentials:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.credentials ports: - "18150:18150" discovery: command: bash -c 'source /edx/app/discovery/discovery_env && while true; do python /edx/app/discovery/discovery/manage.py runserver 0.0.0.0:18381; sleep 2; done' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.discovery" hostname: discovery.devstack.edx depends_on: - mysql @@ -190,10 +158,6 @@ services: ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/discovery:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.discovery ports: - "18381:18381" volumes: @@ -201,6 +165,7 @@ services: ecommerce: command: bash -c 'source /edx/app/ecommerce/ecommerce_env && while true; do python /edx/app/ecommerce/ecommerce/manage.py runserver 0.0.0.0:18130; sleep 2; done' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.ecommerce" hostname: ecommerce.devstack.edx depends_on: - mysql @@ -214,25 +179,18 @@ services: DJANGO_WATCHMAN_TIMEOUT: 30 ENABLE_DJANGO_TOOLBAR: 1 image: edxops/ecommerce:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.ecommerce ports: - "18130:18130" edx_notes_api: command: bash -c 'source /edx/app/edx_notes_api/edx_notes_api_env && while true; do python /edx/app/edx_notes_api/edx_notes_api/manage.py runserver 0.0.0.0:18120 --settings notesserver.settings.devstack; sleep 2; done' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.edxnotesapi" hostname: edx_notes_api.devstack.edx depends_on: - devpi - elasticsearch - mysql image: edxops/notes:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.edxnotesapi ports: - "18120:18120" environment: @@ -248,21 +206,19 @@ services: forum: command: bash -c 'source /edx/app/forum/ruby_env && source /edx/app/forum/devstack_forum_env && cd /edx/app/forum/cs_comments_service && bundle install && while true; do ruby app.rb -o 0.0.0.0 ; sleep 2; done' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.forum" hostname: forum.devstack.edx depends_on: - mongo - memcached - elasticsearch image: edxops/forum:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.forum ports: - "44567:4567" lms: command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack_docker; sleep 2; done' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms" hostname: lms.devstack.edx depends_on: - devpi @@ -284,10 +240,6 @@ services: NO_PYTHON_UNINSTALL: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.lms ports: - "18000:18000" - "19876:19876" # JS test debugging @@ -298,6 +250,7 @@ services: registrar: command: bash -c 'source /edx/app/registrar/registrar_env && while true; do python /edx/app/registrar/registrar/manage.py runserver 0.0.0.0:18734; sleep 2; done' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.registrar" hostname: registrar.devstack.edx depends_on: - discovery @@ -325,10 +278,6 @@ services: CELERY_BROKER_PASSWORD: password DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/registrar:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.registrar ports: - "18734:18734" volumes: @@ -336,6 +285,7 @@ services: registrar-worker: command: bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && celery -A registrar worker -l debug -c 2' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.registrar-worker" hostname: registrar-worker.devstack.edx depends_on: - lms @@ -358,10 +308,6 @@ services: CELERY_BROKER_PASSWORD: password DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/registrar:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.registrar-worker ports: - "18735:18735" volumes: @@ -369,6 +315,7 @@ services: studio: command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py cms runserver 0.0.0.0:18010 --settings devstack_docker; sleep 2; done' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio" hostname: studio.devstack.edx depends_on: - devpi @@ -389,10 +336,6 @@ services: NO_PYTHON_UNINSTALL: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} - networks: - default: - aliases: - - edx.devstack.studio ports: - "18010:18010" - "19877:19877" # JS test debugging @@ -414,10 +357,7 @@ services: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/frontend-app-publisher' - networks: - default: - aliases: - - edx.devstack.frontend-app-publisher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-publisher" ports: - "18400:18400" depends_on: @@ -428,10 +368,7 @@ services: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/gradebook' - networks: - default: - aliases: - - edx.devstack.gradebook + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.gradebook" ports: - "1994:1994" depends_on: @@ -442,10 +379,7 @@ services: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/program-console' - networks: - default: - aliases: - - edx.devstack.program-console + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.program-console" ports: - "1976:1976" depends_on: From ab7b872e05a2d647cf698111deaa38c47c584338 Mon Sep 17 00:00:00 2001 From: "Adolfo R. Brandes" Date: Tue, 19 May 2020 13:54:35 -0300 Subject: [PATCH 04/11] Add network aliases (#2) --- docker-compose-analytics-pipeline.yml | 32 ++++++++++ docker-compose-marketing-site.yml | 4 ++ docker-compose-watchers-nfs.yml | 9 +++ docker-compose-watchers.yml | 8 +++ docker-compose-xqueue.yml | 8 +++ docker-compose.yml | 88 +++++++++++++++++++++++++++ 6 files changed, 149 insertions(+) diff --git a/docker-compose-analytics-pipeline.yml b/docker-compose-analytics-pipeline.yml index f4af248325..5cc50a0438 100644 --- a/docker-compose-analytics-pipeline.yml +++ b/docker-compose-analytics-pipeline.yml @@ -7,6 +7,10 @@ services: hostname: namenode environment: - CLUSTER_NAME=devstack + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.namenode ports: - 127.0.0.1:50070:50070 command: ["/run.sh"] @@ -21,6 +25,10 @@ services: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" depends_on: - namenode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.datanode ports: - 127.0.0.1:50075:50075 command: ["/run.sh"] @@ -40,6 +48,10 @@ services: depends_on: - namenode - datanode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.resourcemanager ports: - 127.0.0.1:8088:8088 # resource manager web ui command: ["/run.sh"] @@ -60,6 +72,10 @@ services: - resourcemanager - namenode - datanode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.nodemanager ports: - 127.0.0.1:8042:8042 # node manager web ui - 127.0.0.1:19888:19888 # node manager job history server ui @@ -69,6 +85,10 @@ services: image: edxops/analytics_pipeline_spark_master:${OPENEDX_RELEASE:-latest} container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkmaster" hostname: sparkmaster + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.sparkmaster ports: - 127.0.0.1:8080:8080 - 127.0.0.1:7077:7077 # spark master port @@ -83,6 +103,10 @@ services: - sparkmaster environment: - SPARK_MASTER=spark://sparkmaster:7077 + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.sparkworker ports: - 127.0.0.1:8081:8081 # spark worker UI @@ -92,6 +116,10 @@ services: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.vertica" volumes: - vertica_data:/home/dbadmin/docker + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.vertica analyticspipeline: image: edxops/analytics_pipeline:${OPENEDX_RELEASE:-latest} @@ -109,6 +137,10 @@ services: - sparkworker - elasticsearch - vertica + networks: + default: + aliases: + - edx.devstack.analytics_pipeline ports: - 127.0.0.1:4040:4040 # spark web UI environment: diff --git a/docker-compose-marketing-site.yml b/docker-compose-marketing-site.yml index 7ff39a2d0d..bc48b89ce1 100644 --- a/docker-compose-marketing-site.yml +++ b/docker-compose-marketing-site.yml @@ -26,5 +26,9 @@ services: # IP address of your machine to enable debugging (IP_ADDRESS set in .env file) - XDEBUG_CONFIG=remote_host=${XDEBUG_IP_ADDRESS:-127.0.0.1} image: edxops/edx-mktg:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.marketing ports: - "8080:80" diff --git a/docker-compose-watchers-nfs.yml b/docker-compose-watchers-nfs.yml index 4998189d31..16d5062a24 100644 --- a/docker-compose-watchers-nfs.yml +++ b/docker-compose-watchers-nfs.yml @@ -14,6 +14,10 @@ services: - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules - src-nfs:/edx/src - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes:cached + networks: + default: + aliases: + - edx.devstack.lms_watcher studio_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' @@ -28,6 +32,11 @@ services: - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules - src-nfs:/edx/src:cached - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes:cached + networks: + default: + aliases: + - edx.devstack.studio_watcher + volumes: edxapp_lms_assets: edxapp_studio_assets: diff --git a/docker-compose-watchers.yml b/docker-compose-watchers.yml index 1746fb2a83..00a37f31d0 100644 --- a/docker-compose-watchers.yml +++ b/docker-compose-watchers.yml @@ -14,6 +14,10 @@ services: - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules - ${DEVSTACK_WORKSPACE}/src:/edx/src:cached - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes:cached + networks: + default: + aliases: + - edx.devstack.lms_watcher studio_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' @@ -28,6 +32,10 @@ services: - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules - ${DEVSTACK_WORKSPACE}/src:/edx/src:cached - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes:cached + networks: + default: + aliases: + - edx.devstack.studio_watcher volumes: edxapp_lms_assets: diff --git a/docker-compose-xqueue.yml b/docker-compose-xqueue.yml index c4f7bf006f..968221d858 100644 --- a/docker-compose-xqueue.yml +++ b/docker-compose-xqueue.yml @@ -8,6 +8,10 @@ services: volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file + networks: + default: + aliases: + - edx.devstack.xqueue ports: - 18040:18040 @@ -17,4 +21,8 @@ services: command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py run_consumer ; sleep 2; done' volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached + networks: + default: + aliases: + - edx.devstack.xqueue_consumer # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file diff --git a/docker-compose.yml b/docker-compose.yml index f997ba854e..be063d71ac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,10 @@ services: hostname: chrome.devstack.edx image: edxops/chrome:${OPENEDX_RELEASE:-latest} shm_size: 2g + networks: + default: + aliases: + - edx.devstack.chrome ports: - "15900:5900" volumes: # for file uploads @@ -31,6 +35,10 @@ services: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.devpi" hostname: devpi.devstack.edx image: edxops/devpi:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.devpi ports: - "3141:3141" volumes: @@ -40,6 +48,10 @@ services: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.elasticsearch" hostname: elasticsearch.devstack.edx image: edxops/elasticsearch:devstack + networks: + default: + aliases: + - edx.devstack.elasticsearch # TODO: What to do about these forwarded ports? They'll conflict with ports forwarded by the Vagrant VM. # ports: # - "9200:9200" @@ -53,12 +65,20 @@ services: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.elasticsearch-5" hostname: elasticsearch-5.devstack.edx image: elasticsearch:5.6.16 + networks: + default: + aliases: + - edx.devstack.elasticsearch-5 firefox: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.firefox" hostname: firefox.devstack.edx image: edxops/firefox:${OPENEDX_RELEASE:-latest} shm_size: 2g + networks: + default: + aliases: + - edx.devstack.firefox ports: - "25900:5900" volumes: # for file uploads @@ -69,6 +89,10 @@ services: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.memcached" hostname: memcached.devstack.edx image: memcached:1.5.10-alpine + networks: + default: + aliases: + - edx.devstack.memcached # ports: # - "11211:11211" @@ -80,6 +104,10 @@ services: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mongo" hostname: mongo.devstack.edx image: mongo:${MONGO_VERSION:-3.6.17} + networks: + default: + aliases: + - edx.devstack.mongo # ports: # - "27017:27017" volumes: @@ -93,6 +121,10 @@ services: MYSQL_ROOT_PASSWORD: "" MYSQL_ALLOW_EMPTY_PASSWORD: "yes" image: mysql:5.6 + networks: + default: + aliases: + - edx.devstack.mysql # ports: # - "3506:3306" volumes: @@ -106,6 +138,10 @@ services: MYSQL_ROOT_PASSWORD: "" MYSQL_ALLOW_EMPTY_PASSWORD: "yes" image: mysql:5.7 + networks: + default: + aliases: + - edx.devstack.mysql57 # ports: # - "3506:3306" volumes: @@ -116,6 +152,10 @@ services: hostname: redis.devstack.edx image: redis:2.8 command: redis-server --requirepass password + networks: + default: + aliases: + - edx.devstack.redis # ================================================ # edX services @@ -139,6 +179,10 @@ services: ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/credentials:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.credentials ports: - "18150:18150" @@ -158,6 +202,10 @@ services: ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/discovery:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.discovery ports: - "18381:18381" volumes: @@ -179,6 +227,10 @@ services: DJANGO_WATCHMAN_TIMEOUT: 30 ENABLE_DJANGO_TOOLBAR: 1 image: edxops/ecommerce:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.ecommerce ports: - "18130:18130" @@ -191,6 +243,10 @@ services: - elasticsearch - mysql image: edxops/notes:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.edxnotesapi ports: - "18120:18120" environment: @@ -213,6 +269,10 @@ services: - memcached - elasticsearch image: edxops/forum:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.forum ports: - "44567:4567" @@ -240,6 +300,10 @@ services: NO_PYTHON_UNINSTALL: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.lms ports: - "18000:18000" - "19876:19876" # JS test debugging @@ -278,6 +342,10 @@ services: CELERY_BROKER_PASSWORD: password DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/registrar:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.registrar ports: - "18734:18734" volumes: @@ -308,6 +376,10 @@ services: CELERY_BROKER_PASSWORD: password DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/registrar:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.registrar-worker ports: - "18735:18735" volumes: @@ -336,6 +408,10 @@ services: NO_PYTHON_UNINSTALL: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.studio ports: - "18010:18010" - "19877:19877" # JS test debugging @@ -358,6 +434,10 @@ services: service: microfrontend working_dir: '/edx/app/frontend-app-publisher' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-publisher" + networks: + default: + aliases: + - edx.devstack.frontend-app-publisher ports: - "18400:18400" depends_on: @@ -369,6 +449,10 @@ services: service: microfrontend working_dir: '/edx/app/gradebook' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.gradebook" + networks: + default: + aliases: + - edx.devstack.gradebook ports: - "1994:1994" depends_on: @@ -380,6 +464,10 @@ services: service: microfrontend working_dir: '/edx/app/program-console' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.program-console" + networks: + default: + aliases: + - edx.devstack.program-console ports: - "1976:1976" depends_on: From 59f812eed18e33f15c1616a0e2176d0aff079d26 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Tue, 19 May 2020 13:56:31 -0400 Subject: [PATCH 05/11] Fixes via new make targets for querying & stopping services * Add dev.print-container. * Add dev.ps * Add dev.stop and dev.stop. * Add dev.kill and dev.kill. * Add dev.down and dev.down. * Add dev.rm-stopped. * Alias old, redundant rules to one of the above. * Via dev.print-container, fix `%-attach` target. * Via dev.print-container, fix update-dbs-init-sql-scrpits.sh. * Via dev.print-container, fix gather-feature-toggle-state.sh. * Via dev.stop & dev.kill, fix docs recommendations for detachment. These new rules are helpful because of PR #523, which adds support for multiple isolated devstacks via overriding COMPOSE_PROJECT_NAME in options.local.mk. Because of that PR, docker-compose commands cannot assume the value of COMPOSE_PROJECT_NAME or the name of containers, so it is even more necessary than before to have useful docker-compose commands exposed through the Makefile. --- Makefile | 58 +++++++++++++++++++++++----------- Makefile.edx | 2 +- README.rst | 12 +++---- docs/devpi.rst | 2 +- gather-feature-toggle-state.sh | 32 +++++++++++-------- provision-e2e.sh | 2 +- update-dbs-init-sql-scripts.sh | 3 +- 7 files changed, 68 insertions(+), 43 deletions(-) mode change 100644 => 100755 gather-feature-toggle-state.sh diff --git a/Makefile b/Makefile index a4542e1e4f..3e48f66bb8 100644 --- a/Makefile +++ b/Makefile @@ -9,11 +9,12 @@ .PHONY: analytics-pipeline-devstack-test analytics-pipeline-shell \ analyticspipeline-shell backup build-courses check-memory \ create-test-course credentials-shell destroy dev.cache-programs \ - dev.check dev.checkout dev.clone dev.clone.ssh dev.nfs.setup \ - devpi-password dev.provision dev.provision.analytics_pipeline \ - dev.provision.services dev.provision.xqueue dev.pull dev.repo.reset \ - dev.reset dev.status dev.sync.daemon.start dev.sync.provision \ - dev.sync.requirements dev.sync.up dev.up dev.up.all \ + dev.check dev.checkout dev.clone dev.clone.ssh dev.down dev.kill \ + dev.nfs.setup devpi-password dev.provision \ + dev.provision.analytics_pipeline dev.provision.services \ + dev.provision.xqueue dev.ps dev.pull dev.repo.reset dev.reset \ + dev.rm-stopped dev.status dev.stop dev.sync.daemon.start \ + dev.sync.provision dev.sync.requirements dev.sync.up dev.up dev.up.all \ dev.up.analytics_pipeline dev.up.watchers dev.up.with-programs \ discovery-shell down e2e-shell e2e-tests ecommerce-shell \ feature-toggle-state forum-restart-devserver healthchecks help \ @@ -110,6 +111,12 @@ upgrade: ## Upgrade requirements with pip-tools pip-compile --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in pip-compile --upgrade -o requirements/base.txt requirements/base.in +dev.print-container.%: ## Get the ID of the running container for a given service. Run with ``make --silent`` for just ID. + @echo -n $$(docker-compose $(DOCKER_COMPOSE_FILES) ps --quiet $*) + +dev.ps: ## View list of created services and their statuses. + docker-compose $(DOCKER_COMPOSE_FILES) ps + dev.checkout: ## Check out "openedx-release/$OPENEDX_RELEASE" in each repo if set, "master" otherwise ./repo.sh checkout @@ -200,22 +207,40 @@ dev.check.%: # Run checks for a given service or set of services (separated by provision: | dev.provision ## This command will be deprecated in a future release, use dev.provision echo "\033[0;31mThis command will be deprecated in a future release, use dev.provision\033[0m" -stop: ## Stop all services +dev.stop: ## Stop all services. + (test -d .docker-sync && docker-sync stop) || true ## Ignore failure here + docker-compose $(DOCKER_COMPOSE_FILES) stop + +dev.stop.%: ## Stop specific services, separated by plus-signs. + docker-compose $(DOCKER_COMPOSE_FILES) stop $$(echo $* | tr + " ") + +stop: dev.stop.$(DEFAULT_SERVICES) + +stop.watchers: dev.stop.lms_watcher+studio_watcher + +stop.all: dev.stop + +stop.xqueue: dev.stop.xqueue+xqueue_consumer + +dev.kill: ## Kill specific services, separated by plus-signs. (test -d .docker-sync && docker-sync stop) || true ## Ignore failure here - docker-compose stop + docker-compose $(DOCKER_COMPOSE_FILES) stop -stop.watchers: ## Stop asset watchers - docker-compose $(DOCKER_COMPOSE_FILES) stop lms_watcher studio_watcher +dev.kill.%: ## Kill specific services, separated by plus-signs. + docker-compose $(DOCKER_COMPOSE_FILES) kill $$(echo $* | tr + " ") -stop.all: | stop.analytics_pipeline stop stop.watchers ## Stop all containers, including asset watchers +dev.rm-stopped: ## Remove stopped containers. Does not affect running containers. + docker-compose $(DOCKER_COMPOSE_FILES) rm --force -stop.xqueue: ## Stop the XQueue and XQueue-Consumer containers - docker-compose $(DOCKER_COMPOSE_FILES) stop xqueue xqueue_consumer +dev.down.%: ## Stop and remove specific services, separated by plus-signs. + docker-compose $(DOCKER_COMPOSE_FILES) rm --force --stop $$(echo $* | tr + " ") -down: ## Remove all service containers and networks +dev.down: ## Stop and remove all service containers and networks (test -d .docker-sync && docker-sync clean) || true ## Ignore failure here docker-compose $(DOCKER_COMPOSE_FILES) down +down: dev.down + destroy: ## Remove all devstack-related containers, networks, and volumes $(WINPTY) bash ./destroy.sh @@ -319,7 +344,7 @@ lms-watcher-shell: ## Run a shell on the LMS watcher container docker-compose exec lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open %-attach: ## Attach to the specified service container process to use the debugger & see logs. - docker-compose attach $* + docker attach "$$(make --silent dev.print-container.$*)" lms-restart: lms-restart-devserver @@ -395,10 +420,7 @@ pull.analytics_pipeline: dev.pull.analyticspipeline ## Update analytics pipeline analytics-pipeline-devstack-test: ## Run analytics pipeline tests in travis build docker-compose $(DOCKER_COMPOSE_FILES) exec -u hadoop -T analyticspipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity' -stop.analytics_pipeline: ## Stop all Analytics pipeline services. - docker-compose $(DOCKER_COMPOSE_FILES) stop \ - namenode datanode resourcemanager nodemanager sparkmaster \ - sparkworker vertica analyticspipeline +stop.analytics_pipeline: dev.stop.namenode+datanode+resourcemanager+nodemanager+sparkmaster+sparkworker+vertica+analyticspipeline ## Stop all Analytics pipeline services. hadoop-application-logs-%: ## View hadoop logs by application Id docker-compose exec nodemanager yarn logs -applicationId $* diff --git a/Makefile.edx b/Makefile.edx index ec0e0cc2a7..87c58d5b25 100644 --- a/Makefile.edx +++ b/Makefile.edx @@ -24,7 +24,7 @@ dev.provision.whitelabel: # AND the test must be setup using the 'dev.provision.whitelabel' target. whitelabel-tests: docker-compose run -T -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e - docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh $(docker-compose ps -q devstack.whitelabel)":/tmp/run_whitelabel_tests.sh + docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh $(make --silent dev.print-container.studio)":/tmp/run_whitelabel_tests.sh docker-compose exec devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh whitelabel-cleanup: diff --git a/README.rst b/README.rst index d3eb1fba0f..c2cb47d3a8 100644 --- a/README.rst +++ b/README.rst @@ -774,20 +774,20 @@ Set a PDB breakpoint anywhere in the code using: and your attached session will offer an interactive PDB prompt when the breakpoint is hit. -To detach from the container, you'll need to stop the container with: +You may be able to detach from the container with the ``Ctrl-P, Ctrl-Q`` key sequence. +If that doesn't work, you will have either close your terminal window, +stop the container with: .. code:: sh - make stop + make dev.stop. -or a manual Docker command to bring down the container: +or kill the container with: .. code:: sh - docker kill $(docker ps -a -q --filter="name=edx.devstack.") + make dev.kill. -Alternatively, some terminals allow detachment from a running container with the -``Ctrl-P, Ctrl-Q`` key sequence. Running LMS and Studio Tests ---------------------------- diff --git a/docs/devpi.rst b/docs/devpi.rst index dc1533b950..b90ae837cd 100644 --- a/docs/devpi.rst +++ b/docs/devpi.rst @@ -60,6 +60,6 @@ Monitoring devpi ---------------- You can monitor the devpi logs by running this command on the host: -``docker logs -f edx.devstack.devpi`` or looking at the output in +``make devpi-logs`` or looking at the output in Kitematic. You can also check the devpi server status by visiting: http://localhost:3141/+status diff --git a/gather-feature-toggle-state.sh b/gather-feature-toggle-state.sh old mode 100644 new mode 100755 index 5c9bce2272..451d3089b8 --- a/gather-feature-toggle-state.sh +++ b/gather-feature-toggle-state.sh @@ -12,20 +12,24 @@ if [ -e feature-toggle-data ]; then fi mkdir feature-toggle-data -docker-compose exec credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py dumpdata waffle --format=json > /edx/app/credentials/credentials/credentials_waffle.json' -docker cp "$(docker-compose ps -q credentials)":/edx/app/credentials/credentials/credentials_waffle.json feature-toggle-data -docker-compose exec credentials bash -c 'rm /edx/app/credentials/credentials/credentials_waffle.json' +credentials_id="$(make --silent dev.print-container.credentials)" +docker exec -t "$credentials_id" bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py dumpdata waffle --format=json > /edx/app/credentials/credentials/credentials_waffle.json' +docker cp "$credentials_id":/edx/app/credentials/credentials/credentials_waffle.json feature-toggle-data +docker exec -t "$credentials_id" bash -c 'rm /edx/app/credentials/credentials/credentials_waffle.json' -docker-compose exec discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py dumpdata waffle --format=json > /edx/app/discovery/discovery/discovery_waffle.json' -docker cp "$(docker-compose ps -q discovery)":/edx/app/discovery/discovery/discovery_waffle.json feature-toggle-data -docker-compose exec discovery bash -c '> /edx/app/discovery/discovery/discovery_waffle.json' +discovery_id="$(make --silent dev.print-container.discovery)" +docker exec -t "$discovery_id" bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py dumpdata waffle --format=json > /edx/app/discovery/discovery/discovery_waffle.json' +docker cp "$discovery_id":/edx/app/discovery/discovery/discovery_waffle.json feature-toggle-data +docker exec -t "$discovery_id" bash -c '> /edx/app/discovery/discovery/discovery_waffle.json' -docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py dumpdata waffle --format=json > /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' -docker cp "$(docker-compose ps -q ecommerce)":/edx/app/ecommerce/ecommerce/ecommerce_waffle.json feature-toggle-data -docker-compose exec ecommerce bash -c 'rm /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' +ecommerce_id="$(make --silent dev.print-container.ecommerce)" +docker exec -t "$ecommerce_id" bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py dumpdata waffle --format=json > /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' +docker cp "$ecommerce_id":/edx/app/ecommerce/ecommerce/ecommerce_waffle.json feature-toggle-data +docker exec -t "$ecommerce_id" bash -c 'rm /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle --format=json > /edx/app/edxapp/edx-platform/lms_waffle.json' -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle_utils --format=json > /edx/app/edxapp/edx-platform/lms_waffle_utils.json' -docker cp "$(docker-compose ps -q lms)":/edx/app/edxapp/edx-platform/lms_waffle.json feature-toggle-data -docker cp "$(docker-compose ps -q lms)":/edx/app/edxapp/edx-platform/lms_waffle_utils.json feature-toggle-data -docker-compose exec lms bash -c 'rm /edx/app/edxapp/edx-platform/lms_waffle*.json' +lms_id="$(make --silent dev.print-container.lms)" +docker exec -t "$lms_id" bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle --format=json > /edx/app/edxapp/edx-platform/lms_waffle.json' +docker exec -t "$lms_id" bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle_utils --format=json > /edx/app/edxapp/edx-platform/lms_waffle_utils.json' +docker cp "$lms_id":/edx/app/edxapp/edx-platform/lms_waffle.json feature-toggle-data +docker cp "$lms_id":/edx/app/edxapp/edx-platform/lms_waffle_utils.json feature-toggle-data +docker exec -t "$lms_id" bash -c 'rm /edx/app/edxapp/edx-platform/lms_waffle*.json' diff --git a/provision-e2e.sh b/provision-e2e.sh index b245bd0d22..86722225d1 100755 --- a/provision-e2e.sh +++ b/provision-e2e.sh @@ -10,7 +10,7 @@ elif [ ! -d "$DEVSTACK_WORKSPACE" ]; then fi # Copy the test course tarball into the studio container -docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz "$(docker-compose ps -q studio)":/tmp/ +docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz "$(make --silent dev.print-container.studio)":/tmp/ # Extract the test course tarball docker-compose exec -T studio bash -c 'cd /tmp && tar xzf course.tar.gz' diff --git a/update-dbs-init-sql-scripts.sh b/update-dbs-init-sql-scripts.sh index 53d9830ee0..456e109a5e 100755 --- a/update-dbs-init-sql-scripts.sh +++ b/update-dbs-init-sql-scripts.sh @@ -24,8 +24,7 @@ make dev.clone.ssh make dev.provision.services.lms+ecommerce # dump schema and data from mysql databases in the mysql docker container and copy them to current directory in docker host -MYSQL_DOCKER_COMPOSE_SERVICE="mysql" -MYSQL_DOCKER_CONTAINER="edx.devstack.${MYSQL_DOCKER_COMPOSE_SERVICE}" +MYSQL_DOCKER_CONTAINER="$(make --silent dev.print-container.mysql)" for DB_NAME in "${DBS[@]}"; do DB_CREATION_SQL_SCRIPT="${DB_NAME}.sql" if [[ " ${EDXAPP_DBS[@]} " =~ " ${DB_NAME} " ]]; then From e161845d2fd4531f7494343d55cfd8c630b63138 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Tue, 26 May 2020 12:57:03 -0400 Subject: [PATCH 06/11] Code review from @ztraboo --- Makefile | 2 +- Makefile.edx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3e48f66bb8..2a4006c4ce 100644 --- a/Makefile +++ b/Makefile @@ -222,7 +222,7 @@ stop.all: dev.stop stop.xqueue: dev.stop.xqueue+xqueue_consumer -dev.kill: ## Kill specific services, separated by plus-signs. +dev.kill: ## Kill all services. (test -d .docker-sync && docker-sync stop) || true ## Ignore failure here docker-compose $(DOCKER_COMPOSE_FILES) stop diff --git a/Makefile.edx b/Makefile.edx index 87c58d5b25..3b4fb3a26e 100644 --- a/Makefile.edx +++ b/Makefile.edx @@ -24,7 +24,7 @@ dev.provision.whitelabel: # AND the test must be setup using the 'dev.provision.whitelabel' target. whitelabel-tests: docker-compose run -T -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e - docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh $(make --silent dev.print-container.studio)":/tmp/run_whitelabel_tests.sh + docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh $(make --silent dev.print-container.devstack.whitelabel)":/tmp/run_whitelabel_tests.sh docker-compose exec devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh whitelabel-cleanup: From 2cc5051c7586269c458bbb40302e23b361a15f09 Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Tue, 26 May 2020 17:14:30 -0400 Subject: [PATCH 07/11] Changes needed for `frontend-app-learning` to multiple release devstack. PR #484 mfe/learning --- docker-compose.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index be063d71ac..01dd47c399 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -479,7 +479,11 @@ services: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/frontend-app-learning' - container_name: edx.devstack.frontend-app-learning + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-learning" + networks: + default: + aliases: + - edx.devstack.frontend-app-learning ports: - "2000:2000" depends_on: From 4f0a62ef324d03b46fab203e3850f96394a501a2 Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Wed, 27 May 2020 11:38:10 -0400 Subject: [PATCH 08/11] Updating `backup` and `restore` Makefile targets to be `docker run` compatible using `dev.print-container.%` Makefile target. The `--volumes-from` option for `docker run` wasn't available for `docker-compose run` so we're reverting this back to previous command. https://docs.docker.com/engine/reference/commandline/run/#mount-volumes-from-container---volumes-from Also the `-n` option for `@echo` command within the `dev.print-container.%` Makefile target was removed because the `$(...)` automatically strips the newline from the command's output, so -n isn't necessary. There was an issue between Mac OS X Catalina and Ubuntu were not outputting the correct `@echo` result from the `dev.print-container.%` target. Mac OS X was including the `-n` option in the echo results while Ubuntu was working properly. @kdmccormick found a fix for this issue. We were looking into alternative platform-agnostic printing commands like `printf`, however, we just stayed with `echo` since ` `$(...)` automatically strips the newline from the command's output. --- Makefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 2a4006c4ce..3e650ce460 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ upgrade: ## Upgrade requirements with pip-tools pip-compile --upgrade -o requirements/base.txt requirements/base.in dev.print-container.%: ## Get the ID of the running container for a given service. Run with ``make --silent`` for just ID. - @echo -n $$(docker-compose $(DOCKER_COMPOSE_FILES) ps --quiet $*) + @echo $$(docker-compose $(DOCKER_COMPOSE_FILES) ps --quiet $*) dev.ps: ## View list of created services and their statuses. docker-compose $(DOCKER_COMPOSE_FILES) ps @@ -291,14 +291,14 @@ validate: ## Validate the devstack configuration docker-compose config backup: dev.up.mysql+mongo+elasticsearch ## Write all data volumes to the host. - docker-compose run --rm --volumes-from -T mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql - docker-compose run --rm --volumes-from -T mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db - docker-compose run --rm --volumes-from -T elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data + docker run --rm --volumes-from $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql + docker run --rm --volumes-from $$(make -s dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db + docker run --rm --volumes-from $$(make -s dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data restore: dev.up.mysql+mongo+elasticsearch ## Restore all data volumes from the host. WARNING: THIS WILL OVERWRITE ALL EXISTING DATA! - docker-compose run --rm --volumes-from -T mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz - docker-compose run --rm --volumes-from -T mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz - docker-compose run --rm --volumes-from -T elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz + docker run --rm --volumes-from $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz + docker run --rm --volumes-from $$(make -s dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz + docker run --rm --volumes-from $$(make -s dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz # TODO: Print out help for this target. Even better if we can iterate over the # services in docker-compose.yml, and print the actual service names. From 4f2253814c4cf1b3249e7af822960634c7370130 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Wed, 27 May 2020 12:00:53 -0400 Subject: [PATCH 09/11] Set COMPOSE_FILE environment variable in Makefile This variable specifies the docker compose YAML files that all calls to `docker-compose` should load. For calls to docker-compose that source from the Makefile, this removes the need to pass "-f file1.yml -f file2.yml ... -f fileN.yml" to docker-compose every single time we invoke it. In the context of PR #532, this commit should fix the xqueue-shell and analyticspipeline-shell commands. --- Makefile | 64 ++++++++++++++++++---------------- provision-analyticspipeline.sh | 8 ++--- provision-credentials.sh | 2 +- provision-forum.sh | 2 +- provision-ida.sh | 2 +- provision-lms.sh | 2 +- provision-marketing.sh | 2 +- provision-registrar.sh | 2 +- provision-xqueue.sh | 8 ++--- 9 files changed, 47 insertions(+), 45 deletions(-) diff --git a/Makefile b/Makefile index 3e650ce460..836464bb3a 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,8 @@ include options.mk -include local.mk # Prefix with hyphen to tolerate absence of file. # Docker Compose YAML files to define services and their volumes. +# This environment variable tells `docker-compose` which files to load definitions +# of services, volumes, and networks from. # Depending on the value of FS_SYNC_STRATEGY, we use a slightly different set of # files, enabling use of different strategies to synchronize files between the host and # the containers. @@ -45,34 +47,34 @@ include options.mk # For example, the LMS/Studio asset watchers are only available for local-mounts and nfs, # and XQueue and the Analytics Pipeline are only available for local-mounts. +# Compose files are separated by a colon. +COMPOSE_PATH_SEPARATOR := : + ifeq ($(FS_SYNC_STRATEGY),local-mounts) -DOCKER_COMPOSE_FILES := \ --f docker-compose-host.yml \ --f docker-compose-themes.yml \ --f docker-compose-watchers.yml \ --f docker-compose-xqueue.yml \ --f docker-compose-analytics-pipeline.yml \ --f docker-compose-marketing-site.yml +COMPOSE_FILE := docker-compose-host.yml +COMPOSE_FILE := $(COMPOSE_FILE):docker-compose-themes.yml +COMPOSE_FILE := $(COMPOSE_FILE):docker-compose-watchers.yml +COMPOSE_FILE := $(COMPOSE_FILE):docker-compose-xqueue.yml +COMPOSE_FILE := $(COMPOSE_FILE):docker-compose-analytics-pipeline.yml +COMPOSE_FILE := $(COMPOSE_FILE):docker-compose-marketing-site.yml endif ifeq ($(FS_SYNC_STRATEGY),nfs) -DOCKER_COMPOSE_FILES := \ --f docker-compose-host-nfs.yml \ --f docker-compose-themes-nfs.yml \ --f docker-compose-watchers-nfs.yml +COMPOSE_FILE := docker-compose-host-nfs.yml +COMPOSE_FILE := $(COMPOSE_FILE):docker-compose-themes-nfs.yml +COMPOSE_FILE := $(COMPOSE_FILE):docker-compose-watchers-nfs.yml endif ifeq ($(FS_SYNC_STRATEGY),docker-sync) -DOCKER_COMPOSE_FILES := \ --f docker-compose-sync.yml +COMPOSE_FILE := docker-compose-sync.yml endif -ifndef DOCKER_COMPOSE_FILES +ifndef COMPOSE_FILE $(error FS_SYNC_STRATEGY is set to $(FS_SYNC_STRATEGY). Must be one of: local-mounts, nfs, docker-sync) endif # All three filesystem synchronization strategy require the main docker-compose.yml file. -DOCKER_COMPOSE_FILES := -f docker-compose.yml $(DOCKER_COMPOSE_FILES) +COMPOSE_FILE := docker-compose.yml:$(COMPOSE_FILE) OS := $(shell uname) @@ -112,10 +114,10 @@ upgrade: ## Upgrade requirements with pip-tools pip-compile --upgrade -o requirements/base.txt requirements/base.in dev.print-container.%: ## Get the ID of the running container for a given service. Run with ``make --silent`` for just ID. - @echo $$(docker-compose $(DOCKER_COMPOSE_FILES) ps --quiet $*) + @echo $$(docker-compose ps --quiet $*) dev.ps: ## View list of created services and their statuses. - docker-compose $(DOCKER_COMPOSE_FILES) ps + docker-compose ps dev.checkout: ## Check out "openedx-release/$OPENEDX_RELEASE" in each repo if set, "master" otherwise ./repo.sh checkout @@ -154,21 +156,21 @@ dev.repo.reset: ## Attempts to reset the local repo checkouts to the master work dev.pull: dev.pull.$(DEFAULT_SERVICES) ## Pull Docker images required by default services. dev.pull.without-deps.%: ## Pull latest Docker images for services (separated by plus-signs). - docker-compose $(DOCKER_COMPOSE_FILES) pull $$(echo $* | tr + " ") + docker-compose pull $$(echo $* | tr + " ") dev.pull.%: ## Pull latest Docker images for services (separated by plus-signs) and all their dependencies. - docker-compose $(DOCKER_COMPOSE_FILES) pull --include-deps $$(echo $* | tr + " ") + docker-compose pull --include-deps $$(echo $* | tr + " ") dev.up: dev.up.$(DEFAULT_SERVICES) check-memory ## Bring up default services. dev.up.%: | check-memory ## Bring up specific services (separated by plus-signs) and their dependencies with host volumes. - docker-compose $(DOCKER_COMPOSE_FILES) up -d $$(echo $* | tr + " ") + docker-compose up -d $$(echo $* | tr + " ") ifeq ($(ALWAYS_CACHE_PROGRAMS),true) make dev.cache-programs endif dev.up.without-deps.%: ## Bring up specific services (separated by plus-signs) without dependencies. - docker-compose $(DOCKER_COMPOSE_FILES) up --d --no-deps $$(echo $* | tr + " ") + docker-compose up --d --no-deps $$(echo $* | tr + " ") dev.up.with-programs: dev.up dev.cache-programs ## Bring up a all services and cache programs in LMS. @@ -209,10 +211,10 @@ provision: | dev.provision ## This command will be deprecated in a future releas dev.stop: ## Stop all services. (test -d .docker-sync && docker-sync stop) || true ## Ignore failure here - docker-compose $(DOCKER_COMPOSE_FILES) stop + docker-compose stop dev.stop.%: ## Stop specific services, separated by plus-signs. - docker-compose $(DOCKER_COMPOSE_FILES) stop $$(echo $* | tr + " ") + docker-compose stop $$(echo $* | tr + " ") stop: dev.stop.$(DEFAULT_SERVICES) @@ -224,20 +226,20 @@ stop.xqueue: dev.stop.xqueue+xqueue_consumer dev.kill: ## Kill all services. (test -d .docker-sync && docker-sync stop) || true ## Ignore failure here - docker-compose $(DOCKER_COMPOSE_FILES) stop + docker-compose stop dev.kill.%: ## Kill specific services, separated by plus-signs. - docker-compose $(DOCKER_COMPOSE_FILES) kill $$(echo $* | tr + " ") + docker-compose kill $$(echo $* | tr + " ") dev.rm-stopped: ## Remove stopped containers. Does not affect running containers. - docker-compose $(DOCKER_COMPOSE_FILES) rm --force + docker-compose rm --force dev.down.%: ## Stop and remove specific services, separated by plus-signs. - docker-compose $(DOCKER_COMPOSE_FILES) rm --force --stop $$(echo $* | tr + " ") + docker-compose rm --force --stop $$(echo $* | tr + " ") dev.down: ## Stop and remove all service containers and networks (test -d .docker-sync && docker-sync clean) || true ## Ignore failure here - docker-compose $(DOCKER_COMPOSE_FILES) down + docker-compose down down: dev.down @@ -245,10 +247,10 @@ destroy: ## Remove all devstack-related containers, networks, and volumes $(WINPTY) bash ./destroy.sh logs: ## View logs from containers running in detached mode - docker-compose $(DOCKER_COMPOSE_FILES) logs -f + docker-compose logs -f %-logs: ## View the logs of the specified service container - docker-compose $(DOCKER_COMPOSE_FILES) logs -f --tail=500 $* + docker-compose logs -f --tail=500 $* RED="\033[0;31m" YELLOW="\033[0;33m" @@ -418,7 +420,7 @@ dev.up.analytics_pipeline: dev.up.analyticspipeline ## Bring up analytics pipeli pull.analytics_pipeline: dev.pull.analyticspipeline ## Update analytics pipeline docker images analytics-pipeline-devstack-test: ## Run analytics pipeline tests in travis build - docker-compose $(DOCKER_COMPOSE_FILES) exec -u hadoop -T analyticspipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity' + docker-compose exec -u hadoop -T analyticspipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity' stop.analytics_pipeline: dev.stop.namenode+datanode+resourcemanager+nodemanager+sparkmaster+sparkworker+vertica+analyticspipeline ## Stop all Analytics pipeline services. diff --git a/provision-analyticspipeline.sh b/provision-analyticspipeline.sh index bb09d437d8..6ee97e2733 100755 --- a/provision-analyticspipeline.sh +++ b/provision-analyticspipeline.sh @@ -17,7 +17,7 @@ elif [ ! -d "$DEVSTACK_WORKSPACE" ]; then fi # Bring the pipeline containers online. -docker-compose $DOCKER_COMPOSE_FILES up -d analyticspipeline +docker-compose up -d analyticspipeline # Analytics pipeline has dependency on lms but we only need its db schema & not full lms. So we'll just load their db # schemas as part of analytics pipeline provisioning. If there is a need of a fully fledged LMS, then provision lms @@ -29,7 +29,7 @@ else echo -e "${GREEN}LMS DB not found, provisioning lms schema.${NC}" docker-compose exec -T mysql bash -c 'mysql -uroot mysql' < provision.sql ./load-db.sh edxapp - docker-compose $DOCKER_COMPOSE_FILES up -d lms studio + docker-compose up -d lms studio docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' #Installing prereqs crashes the process docker-compose restart lms @@ -43,7 +43,7 @@ docker-compose exec -T mysql bash -c 'mysql -uroot mysql' < provision-analytics- # initialize hive metastore echo -e "${GREEN}Initializing HIVE metastore...${NC}" -docker-compose $DOCKER_COMPOSE_FILES exec analyticspipeline bash -c '/edx/app/hadoop/hive/bin/schematool -dbType mysql -initSchema' +docker-compose exec analyticspipeline bash -c '/edx/app/hadoop/hive/bin/schematool -dbType mysql -initSchema' # materialize hadoop directory structure echo -e "${GREEN}Initializing Hadoop directory structure...${NC}" @@ -56,4 +56,4 @@ done sleep 10 # for datanode & other services to activate echo -e "${GREEN}Namenode is ready!${NC}" -docker-compose $DOCKER_COMPOSE_FILES exec -u hadoop analyticspipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' +docker-compose exec -u hadoop analyticspipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' diff --git a/provision-credentials.sh b/provision-credentials.sh index 0334f2966b..a75044dd60 100755 --- a/provision-credentials.sh +++ b/provision-credentials.sh @@ -6,7 +6,7 @@ name=credentials port=18150 -docker-compose $DOCKER_COMPOSE_FILES up -d $name +docker-compose up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" diff --git a/provision-forum.sh b/provision-forum.sh index 6196ac0176..afc51e2827 100755 --- a/provision-forum.sh +++ b/provision-forum.sh @@ -2,5 +2,5 @@ set -e set -o pipefail set -x -docker-compose $DOCKER_COMPOSE_FILES up -d forum +docker-compose up -d forum docker-compose exec -T forum bash -c 'source /edx/app/forum/ruby_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/' diff --git a/provision-ida.sh b/provision-ida.sh index 9ed579ffb5..c830ad2c78 100755 --- a/provision-ida.sh +++ b/provision-ida.sh @@ -4,7 +4,7 @@ client_name=$2 # The name of the Oauth client stored in the edxapp DB. client_port=$3 # The port corresponding to this IDA service in devstack. container_name=${4:-$1} # (Optional) The name of the container. If missing, will use app_name. -docker-compose $DOCKER_COMPOSE_FILES up -d $app_name +docker-compose up -d $app_name echo -e "${GREEN}Installing requirements for ${app_name}...${NC}" docker-compose exec ${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" diff --git a/provision-lms.sh b/provision-lms.sh index 7ef1b8e252..6ec0221867 100755 --- a/provision-lms.sh +++ b/provision-lms.sh @@ -10,7 +10,7 @@ apps=( lms studio ) # Bring edxapp containers online for app in "${apps[@]}"; do - docker-compose $DOCKER_COMPOSE_FILES up -d $app + docker-compose up -d $app done docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' diff --git a/provision-marketing.sh b/provision-marketing.sh index 48bf3f88ca..b7e4417550 100755 --- a/provision-marketing.sh +++ b/provision-marketing.sh @@ -7,7 +7,7 @@ set -x YELLOW='\033[1;33m' NC='\033[0m' # No Color -docker-compose $DOCKER_COMPOSE_FILES up -d marketing +docker-compose up -d marketing set +x echo -e "${YELLOW}edX Marketing Site is not fully provisioned yet.${NC}" diff --git a/provision-registrar.sh b/provision-registrar.sh index c96bfb8843..88d2f6d129 100755 --- a/provision-registrar.sh +++ b/provision-registrar.sh @@ -3,7 +3,7 @@ name=registrar port=18734 -docker-compose $DOCKER_COMPOSE_FILES up -d $name +docker-compose up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" docker-compose exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make requirements' -- "$name" diff --git a/provision-xqueue.sh b/provision-xqueue.sh index 13e0d0869b..7736394a0a 100755 --- a/provision-xqueue.sh +++ b/provision-xqueue.sh @@ -3,11 +3,11 @@ set -o pipefail set -x # Bring up XQueue, we don't need the consumer for provisioning -docker-compose $DOCKER_COMPOSE_FILES up -d xqueue +docker-compose up -d xqueue # Update dependencies -docker-compose $DOCKER_COMPOSE_FILES exec xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && make requirements' +docker-compose exec xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && make requirements' # Run migrations -docker-compose $DOCKER_COMPOSE_FILES exec xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py migrate' +docker-compose exec xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py migrate' # Add users that graders use to fetch data, there's one default user in Ansible which is part of our settings -docker-compose $DOCKER_COMPOSE_FILES exec xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py update_users' +docker-compose exec xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py update_users' From b118b6d322450d3b587044fd25d8b3222265a194 Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Wed, 27 May 2020 15:34:11 -0400 Subject: [PATCH 10/11] Reverting back `e2e` whitelabel commands to be `docker` specific. --- Makefile | 4 ++-- Makefile.edx | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 836464bb3a..3b18f3d30d 100644 --- a/Makefile +++ b/Makefile @@ -320,7 +320,7 @@ ecommerce-shell: ## Run a shell on the ecommerce container docker-compose exec ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open e2e-shell: ## Start the end-to-end tests container with a shell - docker-compose run --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash + docker run -it --network=${COMPOSE_PROJECT_NAME:-devstack}_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash registrar-shell: ## Run a shell on the registrar site container docker-compose exec registrar env TERM=$(TERM) /edx/app/registrar/devstack.sh open @@ -388,7 +388,7 @@ healthchecks: dev.check.$(DEFAULT_SERVICES) healthchecks.%: dev.check.% e2e-tests: ## Run the end-to-end tests against the service containers - docker-compose run --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash -c 'paver e2e_test --exclude="whitelabel\|enterprise"' + docker run -t --network=${COMPOSE_PROJECT_NAME:-devstack}_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash -c 'paver e2e_test --exclude="whitelabel\|enterprise"' validate-lms-volume: ## Validate that changes to the local workspace are reflected in the LMS container touch $(DEVSTACK_WORKSPACE)/edx-platform/testfile diff --git a/Makefile.edx b/Makefile.edx index 3b4fb3a26e..f089acd155 100644 --- a/Makefile.edx +++ b/Makefile.edx @@ -23,12 +23,12 @@ dev.provision.whitelabel: # The containers must be started with the 'dev.up.e2e_wl_tests' target. # AND the test must be setup using the 'dev.provision.whitelabel' target. whitelabel-tests: - docker-compose run -T -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e + docker run -d --name=devstack.whitelabel --network=${COMPOSE_PROJECT_NAME:-devstack}_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh $(make --silent dev.print-container.devstack.whitelabel)":/tmp/run_whitelabel_tests.sh - docker-compose exec devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh + docker exec -t devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh whitelabel-cleanup: - docker-compose rm --force devstack.whitelabel + docker rm --force devstack.whitelabel whitelabel-shell: ## Start a whitelabel test shell session - docker-compose exec devstack.whitelabel env TERM=$(TERM) bash + docker exec -it devstack.whitelabel env TERM=$(TERM) bash From e134c1510a8bd3dae2f5139737b592af3ab0de14 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 28 May 2020 10:00:28 -0400 Subject: [PATCH 11/11] Make sure all uses of docker-compose pass through Makefile Because environment variables are set in options.mk and options.local.mk currently, we cannot assume that docker-compose commands will work currently unless: (1) They are within the Makefile, or (2) They are in a script that is called by the Makefile. Other scripts must use dev.print-container.% to get the ID of a service's container, with which it can then use plain docker commands. Also, this commit adds a dev.restart.% target so that the README does not suggest bare use of docker-compose. Ideally, this situtation should be remedied in the future by moving docker-compose environment variables to a .env file; however, we aren't sure how we would support local overrides in that setup. --- Makefile | 8 +++++++- README.rst | 8 ++++---- destroy.sh | 2 +- in | 27 +++++++++++++++++++++++---- load-db.sh | 3 ++- upgrade_mongo_3_6.sh | 24 +++++++++++++----------- 6 files changed, 50 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 3b18f3d30d..a4dfd3770f 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ dev.nfs.setup devpi-password dev.provision \ dev.provision.analytics_pipeline dev.provision.services \ dev.provision.xqueue dev.ps dev.pull dev.repo.reset dev.reset \ - dev.rm-stopped dev.status dev.stop dev.sync.daemon.start \ + dev.restart dev.rm-stopped dev.status dev.stop dev.sync.daemon.start \ dev.sync.provision dev.sync.requirements dev.sync.up dev.up dev.up.all \ dev.up.analytics_pipeline dev.up.watchers dev.up.with-programs \ discovery-shell down e2e-shell e2e-tests ecommerce-shell \ @@ -224,6 +224,12 @@ stop.all: dev.stop stop.xqueue: dev.stop.xqueue+xqueue_consumer +dev.restart: ## Restart all services. + docker-compose restart $$(echo $* | tr + " ") + +dev.restart.%: ## Restart specific services, separated by plus-signs. + docker-compose restart $$(echo $* | tr + " ") + dev.kill: ## Kill all services. (test -d .docker-sync && docker-sync stop) || true ## Ignore failure here docker-compose stop diff --git a/README.rst b/README.rst index c2cb47d3a8..913b317c0f 100644 --- a/README.rst +++ b/README.rst @@ -405,11 +405,11 @@ already being correcty provisioned. So, when in doubt, it may still be best to run the full ``make dev.provision``. Sometimes you may need to restart a particular application server. To do so, -simply use the ``docker-compose restart`` command: +simply use the ``make dev.restart.%`` command: .. code:: sh - docker-compose restart + make dev.restart. In all the above commands, ```` should be replaced with one of the following: @@ -681,7 +681,7 @@ vary depending on the database. For all of the options, see ``provision.sql``. - Password: ``password`` If you have trouble connecting, ensure the port was mapped successfully by -running ``docker-compose ps`` and looking for a line like this: +running ``make dev.ps`` and looking for a line like this: ``edx.devstack.mysql docker-entrypoint.sh mysql ... Up 0.0.0.0:3506→3306/tcp``. Switching branches @@ -872,7 +872,7 @@ Check the logs If a container stops unexpectedly, you can look at its logs for clues:: - docker-compose logs lms + make -logs Update the code and images ~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/destroy.sh b/destroy.sh index f00ea0f6cc..da47f02cb6 100755 --- a/destroy.sh +++ b/destroy.sh @@ -5,5 +5,5 @@ set -e read -p "This will delete all data in your devstack. Would you like to proceed? [y/n] " -r if [[ $REPLY =~ ^[Yy]$ ]] then - docker-compose -f docker-compose.yml -f docker-compose-watchers.yml -f docker-compose-host.yml -f docker-compose-themes.yml -f docker-compose-analytics-pipeline.yml down -v + docker-compose down -v fi diff --git a/in b/in index 184e21ed1c..e1aa6d8946 100755 --- a/in +++ b/in @@ -1,7 +1,26 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash +# Run a command in a service's container. +# Try to bring up the service if it isn't already up. +# Example: +# ~/devstack> ./in registrar ls /edx/app/ +# edx_ansible nginx registrar supervisor +# Example 2: +# ~/devstack> ./in registrar "cd /edx/app/registrar && ls" +# data registrar registrar.sh venvs +# devstack.sh registrar_env registrar-workers.sh +# nodeenvs registrar_gunicorn.py staticfiles -SERVICE=$1 +set -e +set -o pipefail +set -u + +service="$1" shift -export DEVSTACK_WORKSPACE=${DEVSTACK_WORKSPACE:-$(pwd)/..} -docker-compose -f docker-compose.yml -f docker-compose-host.yml -f docker-compose-themes.yml run $SERVICE exec "$@" +container_id=$(make --silent dev.print-container."$service") +if [[ -z "$container_id" ]]; then + make --silent dev.up."$service" + container_id=$(make --silent dev.print-container."$service") +fi + +docker exec -it "$container_id" bash -c "$*" diff --git a/load-db.sh b/load-db.sh index 7ca21dae64..9b9aaa6c75 100755 --- a/load-db.sh +++ b/load-db.sh @@ -17,5 +17,6 @@ then fi echo "Loading the $1 database..." -docker-compose exec -T mysql bash -c "mysql -uroot $1" < $1.sql +mysql_container=$(make -s dev.print-container.mysql) +docker exec "$mysql_container" bash -c "mysql -uroot $1" < $1.sql echo "Finished loading the $1 database!" diff --git a/upgrade_mongo_3_6.sh b/upgrade_mongo_3_6.sh index e5a0ac4c13..3644a8c7c9 100755 --- a/upgrade_mongo_3_6.sh +++ b/upgrade_mongo_3_6.sh @@ -10,12 +10,13 @@ NC='\033[0m' # No Color export MONGO_VERSION=3.4.24 echo -e "${GREEN}Sarting Mongo ${MONGO_VERSION}${NC}" -docker-compose up -d mongo +make dev.up.mongo +mongo_container="$(make -s dev.print-container.mongo)" echo -e "${GREEN}Waiting for MongoDB...${NC}" -until docker-compose exec -T mongo bash -c 'mongo --eval \"printjson(db.serverStatus())\"' &> /dev/null +until docker exec "$mongo_container" bash -c 'mongo --eval \"printjson(db.serverStatus())\"' &> /dev/null do - if docker-compose logs mongo | grep -q "BadValue: Invalid value for version, found 3.6, expected '3.4' or '3.2'"; then + if docker logs "$mongo_container" | grep -q "BadValue: Invalid value for version, found 3.6, expected '3.4' or '3.2'"; then echo -e "${YELLOW}Already upgraded to Mongo 3.6, exiting${NC}" exit fi @@ -24,15 +25,15 @@ do done echo -e "${GREEN}MongoDB ready.${NC}" -MONGO_VERSION_LIVE=$(docker-compose exec mongo mongo --quiet --eval "printjson(db.version())") -MONGO_VERSION_COMPAT=$(docker-compose exec mongo mongo --quiet \ +MONGO_VERSION_LIVE=$(docker exec -it "$mongo_container" mongo --quiet --eval "printjson(db.version())") +MONGO_VERSION_COMPAT=$(docker exec -it "$mongo_container" mongo --quiet \ --eval "printjson(db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )['featureCompatibilityVersion'])") echo -e "${GREEN}Mongo Server version: ${MONGO_VERSION_LIVE}${NC}" echo -e "${GREEN}Mongo FeatureCompatibilityVersion version: ${MONGO_VERSION_COMPAT}${NC}" if echo "${MONGO_VERSION_COMPAT}" | grep -q "3\.2" ; then echo -e "${GREEN}Upgrading FeatureCompatibilityVersion to 3.4${NC}" - docker-compose exec mongo mongo --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"3.4\" } )" + docker exec -it "$mongo_container" mongo --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"3.4\" } )" else echo -e "${GREEN}FeatureCompatibilityVersion already set to 3.4${NC}" fi @@ -42,25 +43,26 @@ export MONGO_VERSION=3.6.17 echo echo -e "${GREEN}Restarting Mongo on version ${MONGO_VERSION}${NC}" -docker-compose up -d mongo +make dev.up.mongo +mongo_container="$(make -s dev.print-container.mongo)" echo -e "${GREEN}Waiting for MongoDB...${NC}" -until docker-compose exec -T mongo bash -c 'mongo --eval \"printjson(db.serverStatus())\"' &> /dev/null +until docker exec "$mongo_container" bash -c 'mongo --eval \"printjson(db.serverStatus())\"' &> /dev/null do printf "." sleep 1 done echo -e "${GREEN}MongoDB ready.${NC}" -MONGO_VERSION_LIVE=$(docker-compose exec mongo mongo --quiet --eval "printjson(db.version())") -MONGO_VERSION_COMPAT=$(docker-compose exec mongo mongo --quiet \ +MONGO_VERSION_LIVE=$(docker exec -it "$mongo_container" mongo --quiet --eval "printjson(db.version())") +MONGO_VERSION_COMPAT=$(docker exec -it "$mongo_container" mongo --quiet \ --eval "printjson(db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )['featureCompatibilityVersion'])") echo -e "${GREEN}Mongo Server version: ${MONGO_VERSION_LIVE}${NC}" echo -e "${GREEN}Mongo FeatureCompatibilityVersion version: ${MONGO_VERSION_COMPAT}${NC}" if echo "${MONGO_VERSION_COMPAT}" | grep -q "3\.4" ; then echo -e "${GREEN}Upgrading FeatureCompatibilityVersion to 3.6${NC}" - docker-compose exec mongo mongo --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"3.6\" } )" + docker exec -it "$mongo_container" mongo --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"3.6\" } )" else echo -e "${GREEN}FeatureCompatibilityVersion already set to 3.6${NC}" fi