From da3a92f5734ea68f7a45334c578f4cf9bb80ec3d Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Mon, 27 Jul 2020 13:48:19 -0400 Subject: [PATCH 1/5] Update `docker` commands to be `docker-compose` to handle unnamed containers for multi-version devstack support https://discuss.openedx.org/t/docker-devstack-multiple-releases-one-machine/1902 Multiple devstack releases with one machine. This is in regards to this pull request https://github.com/edx/devstack/pull/532 Updating volumes for `themes` to work with the `ironwood` release. Applying these changes to get local devstack working for ironwood release. https://github.com/edx/devstack/pull/376 https://github.com/edx/devstack/pull/377 Moved `COMPOSE_PROJECT_NAME` update to `options.mk` file. Additional changes to get `devstack_docker` for Ironwood working multisite environment setup. Additional changes needed to make `docker` commands work making use of the `dev.print-container.%` target to get the container id. Reverting some of these commands back to `docker` type to align with what we did with https://github.com/edx/devstack/pull/532/commits/59f812eed18e33f15c1616a0e2176d0aff079d26 --- Makefile | 82 ++++++++------- Makefile.edx | 10 +- course-generator/create-courses.sh | 8 +- docker-compose-analytics-pipeline.yml | 48 +++++++-- docker-compose-marketing-site.yml | 8 +- docker-compose-watchers-nfs.yml | 12 ++- docker-compose-watchers.yml | 12 ++- docker-compose-xqueue.yml | 12 ++- docker-compose.yml | 140 +++++++++++++++++++++----- gather-feature-toggle-state.sh | 32 +++--- load-db.sh | 3 +- marketing.mk | 2 +- options.mk | 9 ++ provision-analyticspipeline.sh | 8 +- provision-credentials.sh | 10 +- provision-e2e.sh | 2 +- provision-ecommerce.sh | 6 +- provision-ida-user.sh | 6 +- provision-ida.sh | 8 +- provision-notes.sh | 2 +- provision-registrar.sh | 8 +- provision-retirement-user.sh | 4 +- provision.sh | 8 +- update-dbs-init-sql-scripts.sh | 5 +- upgrade_mongo_3_6.sh | 21 ++-- 25 files changed, 322 insertions(+), 144 deletions(-) diff --git a/Makefile b/Makefile index 1134c14c5e..ff89746093 100644 --- a/Makefile +++ b/Makefile @@ -177,6 +177,10 @@ dev.nfs.%: dev.up.all: dev.up dev.up.watchers ## Bring up all services with host volumes, including watchers +dev.print-container.%: ## Get the ID of the running container for a given service. + @# Can be run as ``make --silent dev.print-container.$service`` for just ID. + @echo $$(docker-compose ps --quiet $*) + # TODO: Improve or rip out Docker Sync targets. # They are not well-fleshed-out and it is not clear if anyone uses them. @@ -265,94 +269,100 @@ pull.xqueue: dev.pull.without-deps.xqueue+xqueue_consumer validate: ## Validate the devstack configuration docker-compose config -backup: dev.up.mysql+mongo+elasticsearch ## Write all data volumes to the host. - docker run --rm --volumes-from edx.devstack.mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql - docker run --rm --volumes-from edx.devstack.mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db - docker run --rm --volumes-from edx.devstack.elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data +backup: ## Write all data volumes to the host. + docker run --rm --volumes-from -T $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql + docker run --rm --volumes-from -T $$(make -s dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db + docker run --rm --volumes-from -T $$(make -s dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data -restore: dev.up.mysql+mongo+elasticsearch ## Restore all data volumes from the host. WARNING: THIS WILL OVERWRITE ALL EXISTING DATA! - docker run --rm --volumes-from edx.devstack.mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz - docker run --rm --volumes-from edx.devstack.mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz - docker run --rm --volumes-from edx.devstack.elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz +restore: ## Restore all data volumes from the host. WARNING: THIS WILL OVERWRITE ALL EXISTING DATA! + docker run --rm --volumes-from -T $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz + docker run --rm --volumes-from -T $$(make -s dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz + docker run --rm --volumes-from -T $$(make -s dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz # TODO: Print out help for this target. Even better if we can iterate over the # services in docker-compose.yml, and print the actual service names. %-shell: ## Run a shell on the specified service container - docker exec -it edx.devstack.$* /bin/bash + docker-compose exec $* /bin/bash analyticspipeline-shell: ## Run a shell on the analytics pipeline container - docker exec -it edx.devstack.analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open + docker-compose exec analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open credentials-shell: ## Run a shell on the credentials container - docker exec -it edx.devstack.credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' + docker-compose exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' discovery-shell: ## Run a shell on the discovery container - docker exec -it edx.devstack.discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open + docker-compose exec discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open ecommerce-shell: ## Run a shell on the ecommerce container - docker exec -it edx.devstack.ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open + docker-compose exec ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open e2e-shell: ## Start the end-to-end tests container with a shell - docker run -it --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash + docker-compose run --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash registrar-shell: ## Run a shell on the registrar site container - docker exec -it edx.devstack.registrar env TERM=$(TERM) /edx/app/registrar/devstack.sh open + docker-compose exec registrar env TERM=$(TERM) /edx/app/registrar/devstack.sh open %-update-db: ## Run migrations for the specified service container - docker exec -t edx.devstack.$* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' + docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' studio-update-db: ## Run migrations for the Studio container - docker exec -t edx.devstack.studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' + docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' lms-update-db: ## Run migrations LMS container - docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' + docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' update-db: | $(DB_MIGRATION_TARGETS) ## Run the migrations for DEFAULT_SERVICES forum-restart-devserver: ## Kill the forum's Sinatra development server. The watcher process will restart it. - docker exec -t edx.devstack.forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")' lms-shell: ## Run a shell on the LMS container - docker exec -it edx.devstack.lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose exec lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open lms-watcher-shell: ## Run a shell on the LMS watcher container - docker exec -it edx.devstack.lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose exec lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open %-attach: ## Attach to the specified service container process to use the debugger & see logs. - docker attach edx.devstack.$* + docker attach "$$(make --silent dev.print-container.$*)" lms-restart: lms-restart-devserver studio-shell: ## Run a shell on the Studio container - docker exec -it edx.devstack.studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose exec studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-watcher-shell: ## Run a shell on the studio watcher container - docker exec -it edx.devstack.studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose exec studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-restart: studio-restart-devserver xqueue-shell: ## Run a shell on the XQueue container - docker exec -it edx.devstack.xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open + docker-compose exec xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue-restart: xqueue-restart-devserver %-restart-devserver: ## Kill a service's Django development server. The watcher process should restart it. - docker exec -t edx.devstack.$* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose exec $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' xqueue_consumer-shell: ## Run a shell on the XQueue consumer container - docker exec -it edx.devstack.xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open + docker-compose exec xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue_consumer-restart: ## Kill the XQueue development server. The watcher process will restart it. - docker exec -t edx.devstack.xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose exec xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")' %-static: ## Rebuild static assets for the specified service container - docker exec -t edx.devstack.$* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' + docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' + +lms-sass: ## Rebuild sass assets for the LMS container + docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=lms --theme-dirs /edx/app/edx-themes/edx-platform/ --themes my-theme' lms-static: ## Rebuild static assets for the LMS container - docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets lms' + docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/ --themes my-theme' + +studio-sass: ## Rebuild sass assets for the Studio container + docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=cms --theme-dirs /edx/app/edx-themes/edx-platform/ --themes my-theme' studio-static: ## Rebuild static assets for the Studio container - docker exec -t edx.devstack.studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets studio' + docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/ --themes my-theme' static: | credentials-static discovery-static ecommerce-static lms-static studio-static ## Rebuild static assets for all service containers @@ -365,12 +375,12 @@ e2e-tests: ## Run the end-to-end tests against the service containers validate-lms-volume: ## Validate that changes to the local workspace are reflected in the LMS container touch $(DEVSTACK_WORKSPACE)/edx-platform/testfile - docker exec edx.devstack.lms ls /edx/app/edxapp/edx-platform/testfile + docker-compose exec -T lms ls /edx/app/edxapp/edx-platform/testfile rm $(DEVSTACK_WORKSPACE)/edx-platform/testfile vnc-passwords: ## Get the VNC passwords for the Chrome and Firefox Selenium containers - @docker logs edx.devstack.chrome 2>&1 | grep "VNC password" | tail -1 - @docker logs edx.devstack.firefox 2>&1 | grep "VNC password" | tail -1 + @docker-compose logs chrome 2>&1 | grep "VNC password" | tail -1 + @docker-compose logs firefox 2>&1 | grep "VNC password" | tail -1 devpi-password: ## Get the root devpi password for the devpi container docker-compose exec devpi bash -c "cat /data/server/.serverpassword" @@ -393,7 +403,7 @@ dev.up.analytics_pipeline: dev.up.analyticspipeline ## Bring up analytics pipeli pull.analytics_pipeline: dev.pull.analyticspipeline ## Update analytics pipeline docker images analytics-pipeline-devstack-test: ## Run analytics pipeline tests in travis build - docker exec -u hadoop -i edx.devstack.analytics_pipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity' + docker-compose $(DOCKER_COMPOSE_FILES) exec -u hadoop -T analytics_pipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity' stop.analytics_pipeline: ## Stop all Analytics pipeline services. docker-compose $(DOCKER_COMPOSE_FILES) stop \ @@ -401,7 +411,7 @@ stop.analytics_pipeline: ## Stop all Analytics pipeline services. sparkworker vertica analyticspipeline hadoop-application-logs-%: ## View hadoop logs by application Id - docker exec -it edx.devstack.analytics_pipeline.nodemanager yarn logs -applicationId $* + docker-compose exec analytics_pipeline.nodemanager yarn logs -applicationId $* # Provisions studio, ecommerce, and marketing with course(s) in test-course.json # Modify test-course.json before running this make target to generate a custom course diff --git a/Makefile.edx b/Makefile.edx index 81343ba2de..030273960e 100644 --- a/Makefile.edx +++ b/Makefile.edx @@ -23,12 +23,12 @@ dev.provision.whitelabel: # The containers must be started with the 'dev.up.e2e_wl_tests' target. # AND the test must be setup using the 'dev.provision.whitelabel' target. whitelabel-tests: - docker run -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e - docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh devstack.whitelabel:/tmp/run_whitelabel_tests.sh - docker exec -t devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh + docker-compose run -T -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e + docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh $(make --silent dev.print-container.devstack.whitelabel)":/tmp/run_whitelabel_tests.sh + docker-compose exec devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh whitelabel-cleanup: - docker rm --force devstack.whitelabel + docker-compose rm --force devstack.whitelabel whitelabel-shell: ## Start a whitelabel test shell session - docker exec -it devstack.whitelabel env TERM=$(TERM) bash + docker-compose exec -it devstack.whitelabel env TERM=$(TERM) bash diff --git a/course-generator/create-courses.sh b/course-generator/create-courses.sh index eea96f6589..b7ee059a9e 100755 --- a/course-generator/create-courses.sh +++ b/course-generator/create-courses.sh @@ -15,14 +15,14 @@ for arg in "$@"; do studio=true fi elif [ $arg == "--ecommerce" ]; then - if [ ! "$(docker exec -t edx.devstack.ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then + if [ ! "$(docker-compose exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then echo "Issue with ecommerce container" container_error=true else ecommerce=true fi elif [ $arg == "--marketing" ]; then - if [ ! "$(docker exec -t edx.devstack.marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then + if [ ! "$(docker-compose exec marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then echo "Issue with marketing container. Course creation will proceed without marketing container." else marketing=true @@ -54,10 +54,10 @@ fi if $ecommerce ; then echo "Creating courses on ecommerce." - docker exec -t edx.devstack.ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" + docker-compose exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" fi if $marketing ; then echo "Creating courses on marketing." - docker exec -t edx.devstack.marketing bash -c "drush generate_courses '$course_json'" + docker-compose exec marketing bash -c "drush generate_courses '$course_json'" fi diff --git a/docker-compose-analytics-pipeline.yml b/docker-compose-analytics-pipeline.yml index aa84af84a6..01b9016acf 100644 --- a/docker-compose-analytics-pipeline.yml +++ b/docker-compose-analytics-pipeline.yml @@ -2,11 +2,15 @@ version: "2.1" services: namenode: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.namenode" image: edxops/analytics_pipeline_hadoop_namenode:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.namenode hostname: namenode environment: - CLUSTER_NAME=devstack + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.namenode ports: - 127.0.0.1:50070:50070 command: ["/run.sh"] @@ -14,13 +18,17 @@ services: - namenode_data:/hadoop/dfs/name datanode: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.datanode" image: edxops/analytics_pipeline_hadoop_datanode:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.datanode hostname: datanode environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" depends_on: - namenode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.datanode ports: - 127.0.0.1:50075:50075 command: ["/run.sh"] @@ -28,8 +36,8 @@ services: - datanode_data:/hadoop/dfs/data resourcemanager: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.resourcemanager" image: edxops/analytics_pipeline_hadoop_resourcemanager:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.resourcemanager hostname: resourcemanager environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" @@ -40,13 +48,17 @@ services: depends_on: - namenode - datanode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.resourcemanager ports: - 127.0.0.1:8088:8088 # resource manager web ui command: ["/run.sh"] nodemanager: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.nodemanager" image: edxops/analytics_pipeline_hadoop_nodemanager:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.nodemanager hostname: nodemanager environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" @@ -60,15 +72,23 @@ services: - resourcemanager - namenode - datanode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.nodemanager ports: - 127.0.0.1:8042:8042 # node manager web ui - 127.0.0.1:19888:19888 # node manager job history server ui command: ["/run.sh"] sparkmaster: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkmaster" image: edxops/analytics_pipeline_spark_master:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.sparkmaster hostname: sparkmaster + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.sparkmaster ports: - 127.0.0.1:8080:8080 - 127.0.0.1:7077:7077 # spark master port @@ -76,26 +96,34 @@ services: - 127.0.0.1:18080:18080 # spark history server sparkworker: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkworker" image: edxops/analytics_pipeline_spark_worker:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.sparkworker hostname: sparkworker depends_on: - sparkmaster environment: - SPARK_MASTER=spark://sparkmaster:7077 + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.sparkworker ports: - 127.0.0.1:8081:8081 # spark worker UI vertica: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.vertica" image: iamamr/vertica:9.1.0-0 hostname: vertica - container_name: edx.devstack.analytics_pipeline.vertica + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.vertica volumes: - vertica_data:/home/dbadmin/docker analyticspipeline: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline" image: edxops/analytics_pipeline:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline hostname: analyticspipeline volumes: - ${DEVSTACK_WORKSPACE}/edx-analytics-pipeline:/edx/app/analytics_pipeline/analytics_pipeline @@ -109,6 +137,10 @@ services: - sparkworker - elasticsearch - vertica + networks: + default: + aliases: + - edx.devstack.analytics_pipeline ports: - 127.0.0.1:4040:4040 # spark web UI environment: diff --git a/docker-compose-marketing-site.yml b/docker-compose-marketing-site.yml index 1051b4f872..bc48b89ce1 100644 --- a/docker-compose-marketing-site.yml +++ b/docker-compose-marketing-site.yml @@ -7,7 +7,7 @@ services: - MARKETING_SITE_ROOT="http://localhost:8080" marketing: - container_name: edx.devstack.marketing + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.marketing" depends_on: - mysql - memcached @@ -25,6 +25,10 @@ services: - DRUPAL_EXTRA_SETTINGS=${DRUPAL_EXTRA_SETTINGS:-/var/www/html/sites/default/docker.settings.php} # IP address of your machine to enable debugging (IP_ADDRESS set in .env file) - XDEBUG_CONFIG=remote_host=${XDEBUG_IP_ADDRESS:-127.0.0.1} - image: edxops/edx-mktg:latest + image: edxops/edx-mktg:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.marketing ports: - "8080:80" diff --git a/docker-compose-watchers-nfs.yml b/docker-compose-watchers-nfs.yml index 532a49060f..92fe18bdee 100644 --- a/docker-compose-watchers-nfs.yml +++ b/docker-compose-watchers-nfs.yml @@ -3,11 +3,15 @@ version: "2.1" services: lms_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.lms_watcher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.lms_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.lms_watcher volumes: - edx-nfs:/edx/app/edxapp/edx-platform - edxapp_lms_assets:/edx/var/edxapp/staticfiles/ @@ -17,11 +21,15 @@ services: studio_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.studio_watcher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.studio_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.studio_watcher volumes: - edxapp_studio_assets:/edx/var/edxapp/staticfiles/ - edx-nfs:/edx/app/edxapp/edx-platform diff --git a/docker-compose-watchers.yml b/docker-compose-watchers.yml index 7c7828f701..006d709d9d 100644 --- a/docker-compose-watchers.yml +++ b/docker-compose-watchers.yml @@ -3,11 +3,15 @@ version: "2.1" services: lms_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.lms_watcher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.lms_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.lms_watcher volumes: - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform:cached - edxapp_lms_assets:/edx/var/edxapp/staticfiles/ @@ -17,11 +21,15 @@ services: studio_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.studio_watcher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.studio_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.studio_watcher volumes: - edxapp_studio_assets:/edx/var/edxapp/staticfiles/ - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform:cached diff --git a/docker-compose-xqueue.yml b/docker-compose-xqueue.yml index bf4a73ac54..89e987c5ba 100644 --- a/docker-compose-xqueue.yml +++ b/docker-compose-xqueue.yml @@ -2,19 +2,27 @@ version: "2.1" services: xqueue: - container_name: edx.devstack.xqueue + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue" image: edxops/xqueue:${OPENEDX_RELEASE:-latest} command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py runserver 0.0.0.0:18040 ; sleep 2; done' volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file + networks: + default: + aliases: + - edx.devstack.xqueue ports: - 18040:18040 xqueue_consumer: - container_name: edx.devstack.xqueue_consumer + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue_consumer" image: edxops/xqueue:${OPENEDX_RELEASE:-latest} command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py run_consumer ; sleep 2; done' + networks: + default: + aliases: + - edx.devstack.xqueue_consumer volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file diff --git a/docker-compose.yml b/docker-compose.yml index 122ef80290..53bd811ba7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,10 +17,14 @@ services: # ================================================ chrome: - container_name: edx.devstack.chrome + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.chrome" hostname: chrome.devstack.edx image: edxops/chrome:${OPENEDX_RELEASE:-latest} shm_size: 2g + networks: + default: + aliases: + - edx.devstack.chrome ports: - "15900:5900" volumes: # for file uploads @@ -28,18 +32,26 @@ services: - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data devpi: - container_name: edx.devstack.devpi + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.devpi" hostname: devpi.devstack.edx image: edxops/devpi:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.devpi ports: - "3141:3141" volumes: - devpi_data:/data elasticsearch: - container_name: edx.devstack.elasticsearch + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.elasticsearch" hostname: elasticsearch.devstack.edx image: edxops/elasticsearch:devstack + networks: + default: + aliases: + - edx.devstack.elasticsearch # TODO: What to do about these forwarded ports? They'll conflict with ports forwarded by the Vagrant VM. # ports: # - "9200:9200" @@ -50,15 +62,23 @@ services: # This is meant to be used to test ES upgrades so that we do not have to upgrade all of our services to ES5 at once. elasticsearch-5: - container_name: edx.devstack.elasticsearch-5 + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.elasticsearch-5" hostname: elasticsearch-5.devstack.edx image: elasticsearch:5.6.16 + networks: + default: + aliases: + - edx.devstack.elasticsearch-5 firefox: - container_name: edx.devstack.firefox + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.firefox" hostname: firefox.devstack.edx image: edxops/firefox:${OPENEDX_RELEASE:-latest} shm_size: 2g + networks: + default: + aliases: + - edx.devstack.firefox ports: - "25900:5900" volumes: # for file uploads @@ -66,9 +86,13 @@ services: - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data memcached: - container_name: edx.devstack.memcached + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.memcached" hostname: memcached.devstack.edx image: memcached:1.5.10-alpine + networks: + default: + aliases: + - edx.devstack.memcached # ports: # - "11211:11211" @@ -77,9 +101,13 @@ services: # to conserve disk space, and disable the journal for a minor performance gain. # See https://docs.mongodb.com/v3.0/reference/program/mongod/#options for complete details. command: mongod --smallfiles --nojournal --storageEngine wiredTiger - container_name: edx.devstack.mongo + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mongo" hostname: mongo.devstack.edx image: mongo:${MONGO_VERSION:-3.6.17} + networks: + default: + aliases: + - edx.devstack.mongo # ports: # - "27017:27017" volumes: @@ -87,34 +115,46 @@ services: mysql: command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci - container_name: edx.devstack.mysql + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql" hostname: mysql.devstack.edx environment: MYSQL_ROOT_PASSWORD: "" MYSQL_ALLOW_EMPTY_PASSWORD: "yes" image: mysql:5.6 + networks: + default: + aliases: + - edx.devstack.mysql # ports: - # - "3506:3306" + # - "3506:3306" volumes: - mysql_data:/var/lib/mysql mysql57: command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci - container_name: edx.devstack.mysql57 + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql57" hostname: mysql57.devstack.edx environment: MYSQL_ROOT_PASSWORD: "" MYSQL_ALLOW_EMPTY_PASSWORD: "yes" image: mysql:5.7 + networks: + default: + aliases: + - edx.devstack.mysql57 # ports: # - "3506:3306" volumes: - mysql57_data:/var/lib/mysql redis: - container_name: edx.devstack.redis + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.redis" hostname: redis.devstack.edx image: redis:2.8 + networks: + default: + aliases: + - edx.devstack.redis command: redis-server --requirepass password # ================================================ @@ -123,7 +163,7 @@ services: credentials: command: bash -c 'source /edx/app/credentials/credentials_env && while true; do python /edx/app/credentials/credentials/manage.py runserver 0.0.0.0:18150; sleep 2; done' - container_name: edx.devstack.credentials + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.credentials" hostname: credentials.devstack.edx depends_on: - mysql @@ -139,12 +179,16 @@ services: ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/credentials:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.credentials ports: - "18150:18150" discovery: command: bash -c 'source /edx/app/discovery/discovery_env && while true; do python /edx/app/discovery/discovery/manage.py runserver 0.0.0.0:18381; sleep 2; done' - container_name: edx.devstack.discovery + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.discovery" hostname: discovery.devstack.edx depends_on: - mysql @@ -158,6 +202,10 @@ services: ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/discovery:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.discovery ports: - "18381:18381" volumes: @@ -165,7 +213,7 @@ services: ecommerce: command: bash -c 'source /edx/app/ecommerce/ecommerce_env && while true; do python /edx/app/ecommerce/ecommerce/manage.py runserver 0.0.0.0:18130; sleep 2; done' - container_name: edx.devstack.ecommerce + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.ecommerce" hostname: ecommerce.devstack.edx depends_on: - mysql @@ -179,18 +227,26 @@ services: DJANGO_WATCHMAN_TIMEOUT: 30 ENABLE_DJANGO_TOOLBAR: 1 image: edxops/ecommerce:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.ecommerce ports: - "18130:18130" edx_notes_api: command: bash -c 'source /edx/app/edx_notes_api/edx_notes_api_env && while true; do python /edx/app/edx_notes_api/edx_notes_api/manage.py runserver 0.0.0.0:18120 --settings notesserver.settings.devstack; sleep 2; done' - container_name: edx.devstack.edxnotesapi + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.edx_notes_api" hostname: edx_notes_api.devstack.edx depends_on: - devpi - elasticsearch - mysql image: edxops/notes:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.edxnotesapi ports: - "18120:18120" environment: @@ -206,19 +262,23 @@ services: forum: command: bash -c 'source /edx/app/forum/ruby_env && source /edx/app/forum/devstack_forum_env && cd /edx/app/forum/cs_comments_service && bundle install && while true; do ruby app.rb -o 0.0.0.0 ; sleep 2; done' - container_name: edx.devstack.forum + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.forum" hostname: forum.devstack.edx depends_on: - mongo - memcached - elasticsearch image: edxops/forum:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.forum ports: - "44567:4567" lms: command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack_docker; sleep 2; done' - container_name: edx.devstack.lms + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms" hostname: lms.devstack.edx depends_on: - devpi @@ -240,6 +300,10 @@ services: NO_PYTHON_UNINSTALL: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.lms ports: - "18000:18000" - "19876:19876" # JS test debugging @@ -250,7 +314,7 @@ services: registrar: command: bash -c 'source /edx/app/registrar/registrar_env && while true; do python /edx/app/registrar/registrar/manage.py runserver 0.0.0.0:18734; sleep 2; done' - container_name: edx.devstack.registrar + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.registrar" hostname: registrar.devstack.edx depends_on: - discovery @@ -278,6 +342,10 @@ services: CELERY_BROKER_PASSWORD: password DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/registrar:latest + networks: + default: + aliases: + - edx.devstack.registrar ports: - "18734:18734" volumes: @@ -285,7 +353,7 @@ services: registrar-worker: command: bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && celery -A registrar worker -l debug -c 2' - container_name: edx.devstack.registrar-worker + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.registrar-worker" hostname: registrar-worker.devstack.edx depends_on: - lms @@ -308,6 +376,10 @@ services: CELERY_BROKER_PASSWORD: password DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/registrar:latest + networks: + default: + aliases: + - edx.devstack.registrar-worker ports: - "18735:18735" volumes: @@ -315,7 +387,7 @@ services: studio: command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py cms runserver 0.0.0.0:18010 --settings devstack_docker; sleep 2; done' - container_name: edx.devstack.studio + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio" hostname: studio.devstack.edx depends_on: - devpi @@ -336,6 +408,10 @@ services: NO_PYTHON_UNINSTALL: 1 DJANGO_WATCHMAN_TIMEOUT: 30 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.studio ports: - "18010:18010" - "19877:19877" # JS test debugging @@ -353,33 +429,45 @@ services: # ========================================================================== frontend-app-publisher: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-publisher" extends: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/frontend-app-publisher' - container_name: edx.devstack.frontend-app-publisher + networks: + default: + aliases: + - edx.devstack.frontend-app-publisher ports: - "18400:18400" depends_on: - lms gradebook: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.gradebook" extends: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/gradebook' - container_name: edx.devstack.gradebook + networks: + default: + aliases: + - edx.devstack.gradebook ports: - "1994:1994" depends_on: - lms program-console: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.program-console" extends: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/program-console' - container_name: edx.devstack.program-console + networks: + default: + aliases: + - edx.devstack.program-console ports: - "1976:1976" depends_on: @@ -387,11 +475,15 @@ services: - registrar frontend-app-learning: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.frontend-app-learning" extends: file: microfrontend.yml service: microfrontend working_dir: '/edx/app/frontend-app-learning' - container_name: edx.devstack.frontend-app-learning + networks: + default: + aliases: + - edx.devstack.frontend-app-learning ports: - "2000:2000" depends_on: diff --git a/gather-feature-toggle-state.sh b/gather-feature-toggle-state.sh index 2ed9946195..c78a73642b 100644 --- a/gather-feature-toggle-state.sh +++ b/gather-feature-toggle-state.sh @@ -12,20 +12,24 @@ if [ -e feature-toggle-data ]; then fi mkdir feature-toggle-data -docker exec -t edx.devstack.credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py dumpdata waffle --format=json > /edx/app/credentials/credentials/credentials_waffle.json' -docker cp edx.devstack.credentials:/edx/app/credentials/credentials/credentials_waffle.json feature-toggle-data -docker exec -t edx.devstack.credentials bash -c 'rm /edx/app/credentials/credentials/credentials_waffle.json' +credentials_id="$(make --silent dev.print-container.credentials)" +docker exec -t "$credentials_id" bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py dumpdata waffle --format=json > /edx/app/credentials/credentials/credentials_waffle.json' +docker cp "$credentials_id":/edx/app/credentials/credentials/credentials_waffle.json feature-toggle-data +docker exec -t "$credentials_id" bash -c 'rm /edx/app/credentials/credentials/credentials_waffle.json' -docker exec -t edx.devstack.discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py dumpdata waffle --format=json > /edx/app/discovery/discovery/discovery_waffle.json' -docker cp edx.devstack.discovery:/edx/app/discovery/discovery/discovery_waffle.json feature-toggle-data -docker exec -t edx.devstack.discovery bash -c '> /edx/app/discovery/discovery/discovery_waffle.json' +discovery_id="$(make --silent dev.print-container.discovery)" +docker exec -t "$discovery_id" bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py dumpdata waffle --format=json > /edx/app/discovery/discovery/discovery_waffle.json' +docker cp "$discovery_id".discovery:/edx/app/discovery/discovery/discovery_waffle.json feature-toggle-data +docker exec -t "$discovery_id" bash -c '> /edx/app/discovery/discovery/discovery_waffle.json' -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py dumpdata waffle --format=json > /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' -docker cp edx.devstack.ecommerce:/edx/app/ecommerce/ecommerce/ecommerce_waffle.json feature-toggle-data -docker exec -t edx.devstack.ecommerce bash -c 'rm /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' +ecommerce_id="$(make --silent dev.print-container.ecommerce)" +docker exec -t "$ecommerce_id" bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py dumpdata waffle --format=json > /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' +docker cp "$ecommerce_id".ecommerce:/edx/app/ecommerce/ecommerce/ecommerce_waffle.json feature-toggle-data +docker exec -t "$ecommerce_id" bash -c 'rm /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle --format=json > /edx/app/edxapp/edx-platform/lms_waffle.json' -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle_utils --format=json > /edx/app/edxapp/edx-platform/lms_waffle_utils.json' -docker cp edx.devstack.lms:/edx/app/edxapp/edx-platform/lms_waffle.json feature-toggle-data -docker cp edx.devstack.lms:/edx/app/edxapp/edx-platform/lms_waffle_utils.json feature-toggle-data -docker exec -t edx.devstack.lms bash -c 'rm /edx/app/edxapp/edx-platform/lms_waffle*.json' +lms_id="$(make --silent dev.print-container.lms)" +docker exec -t "$lms_id" bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle --format=json > /edx/app/edxapp/edx-platform/lms_waffle.json' +docker exec -t "$lms_id" bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle_utils --format=json > /edx/app/edxapp/edx-platform/lms_waffle_utils.json' +docker cp "$lms_id":/edx/app/edxapp/edx-platform/lms_waffle.json feature-toggle-data +docker cp "$lms_id":/edx/app/edxapp/edx-platform/lms_waffle_utils.json feature-toggle-data +docker exec -t "$lms_id" bash -c 'rm /edx/app/edxapp/edx-platform/lms_waffle*.json' diff --git a/load-db.sh b/load-db.sh index 24b70fada5..9b9aaa6c75 100755 --- a/load-db.sh +++ b/load-db.sh @@ -17,5 +17,6 @@ then fi echo "Loading the $1 database..." -docker exec -i edx.devstack.mysql mysql -uroot $1 < $1.sql +mysql_container=$(make -s dev.print-container.mysql) +docker exec "$mysql_container" bash -c "mysql -uroot $1" < $1.sql echo "Finished loading the $1 database!" diff --git a/marketing.mk b/marketing.mk index 16a8adb4a2..c1496c3ef0 100644 --- a/marketing.mk +++ b/marketing.mk @@ -3,7 +3,7 @@ help-marketing: ## Display this help message @awk -F ':.*?## ' '/^[a-zA-Z]/ && NF==2 {printf "\033[36m %-25s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | grep marketing | sort marketing-shell: ## Run a shell on the marketing site container - docker exec -it edx.devstack.marketing env TERM=$(TERM) bash -c 'cd /edx/app/edx-mktg/edx-mktg; exec /bin/bash -sh' + docker-compose exec marketing env TERM=$(TERM) bash -c 'cd /edx/app/edx-mktg/edx-mktg; exec /bin/bash -sh' stop-marketing: ## Stop all services (including the marketing site) with host volumes docker-compose -f docker-compose.yml -f docker-compose-host.yml -f docker-compose-themes.yml -f docker-compose-marketing-site.yml -f docker-compose-marketing-site-host.yml stop diff --git a/options.mk b/options.mk index d0f98018b7..0e0a456099 100644 --- a/options.mk +++ b/options.mk @@ -8,6 +8,15 @@ # Defaults to parent. DEVSTACK_WORKSPACE ?= $(shell pwd)/.. +# Name of Docker Compose project. +# See https://docs.docker.com/compose/reference/envvars/#compose_project_name +# Defaults to 'devstack' should OPENEDX_RELEASE not be defined. +ifdef OPENEDX_RELEASE + COMPOSE_PROJECT_NAME ?= devstack-${OPENEDX_RELEASE} +else + COMPOSE_PROJECT_NAME ?= devstack +endif + # Name of Docker Compose project. # See https://docs.docker.com/compose/reference/envvars/#compose_project_name # Defaults to 'devstack'. diff --git a/provision-analyticspipeline.sh b/provision-analyticspipeline.sh index 70e458e683..0fd4d7c81a 100755 --- a/provision-analyticspipeline.sh +++ b/provision-analyticspipeline.sh @@ -22,12 +22,12 @@ docker-compose $DOCKER_COMPOSE_FILES up -d analyticspipeline # Analytics pipeline has dependency on lms but we only need its db schema & not full lms. So we'll just load their db # schemas as part of analytics pipeline provisioning. If there is a need of a fully fledged LMS, then provision lms # by following their documentation. -if [[ ! -z "`docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='edxapp'" 2>&1`" ]]; +if [[ ! -z "`docker-compose exec mysql mysql -uroot -se "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='edxapp'" 2>&1`" ]]; then echo -e "${GREEN}LMS DB exists, skipping lms schema load.${NC}" else echo -e "${GREEN}LMS DB not found, provisioning lms schema.${NC}" - docker exec -i edx.devstack.mysql mysql -uroot mysql < provision.sql + docker-compose exec mysql bash -c 'mysql -uroot mysql' < provision.sql ./load-db.sh edxapp docker-compose $DOCKER_COMPOSE_FILES up -d lms studio docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' @@ -39,7 +39,7 @@ fi echo -e "${GREEN}LMS database provisioned successfully...${NC}" echo -e "${GREEN}Creating databases and users...${NC}" -docker exec -i edx.devstack.mysql mysql -uroot mysql < provision-analytics-pipeline.sql +docker-compose exec mysql bash -c 'mysql -uroot mysql' < provision-analytics-pipeline.sql # initialize hive metastore echo -e "${GREEN}Initializing HIVE metastore...${NC}" @@ -56,4 +56,4 @@ done sleep 10 # for datanode & other services to activate echo -e "${GREEN}Namenode is ready!${NC}" -docker exec -u hadoop -i edx.devstack.analytics_pipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' +docker-compose exec -u hadoop analytics_pipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' diff --git a/provision-credentials.sh b/provision-credentials.sh index 32fccdaf61..0334f2966b 100755 --- a/provision-credentials.sh +++ b/provision-credentials.sh @@ -9,20 +9,20 @@ port=18150 docker-compose $DOCKER_COMPOSE_FILES up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" echo -e "${GREEN}Configuring site for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' +docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker-compose exec ${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-e2e.sh b/provision-e2e.sh index b9e7b5508e..86722225d1 100755 --- a/provision-e2e.sh +++ b/provision-e2e.sh @@ -10,7 +10,7 @@ elif [ ! -d "$DEVSTACK_WORKSPACE" ]; then fi # Copy the test course tarball into the studio container -docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz edx.devstack.studio:/tmp/ +docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz "$(make --silent dev.print-container.studio)":/tmp/ # Extract the test course tarball docker-compose exec -T studio bash -c 'cd /tmp && tar xzf course.tar.gz' diff --git a/provision-ecommerce.sh b/provision-ecommerce.sh index c1680fab02..00068b6c0e 100755 --- a/provision-ecommerce.sh +++ b/provision-ecommerce.sh @@ -4,6 +4,6 @@ ./provision-ida.sh ecommerce ecommerce 18130 # Configure ecommerce -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/' -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' +docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --client-id=ecommerce-key --client-secret=ecommerce-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --journals_api_url=http://journals.app:18606/api/v1/' +docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' +docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' diff --git a/provision-ida-user.sh b/provision-ida-user.sh index 1a3b93c9ee..965de8a566 100755 --- a/provision-ida-user.sh +++ b/provision-ida-user.sh @@ -7,8 +7,8 @@ client_port=$3 echo -e "${GREEN}Creating service user and OAuth2 applications for ${app_name}...${NC}" # Create the service user. -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" # Create the DOT applications - one for single sign-on and one for backend service IDA-to-IDA authentication. -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "http://localhost:$3/complete/edx-oauth2/" --client-id "$1-sso-key" --client-secret "$1-sso-secret" --scopes "user_id" $1-sso $1_worker' -- "$app_name" "$client_name" "$client_port" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "$1-backend-service-key" --client-secret "$1-backend-service-secret" $1-backend-service $1_worker' -- "$app_name" "$client_name" "$client_port" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "http://localhost:$3/complete/edx-oauth2/" --client-id "$1-sso-key" --client-secret "$1-sso-secret" --scopes "user_id" $1-sso $1_worker' -- "$app_name" "$client_name" "$client_port" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "$1-backend-service-key" --client-secret "$1-backend-service-secret" $1-backend-service $1_worker' -- "$app_name" "$client_name" "$client_port" diff --git a/provision-ida.sh b/provision-ida.sh index 62bbf4d0ad..b1f967d121 100755 --- a/provision-ida.sh +++ b/provision-ida.sh @@ -7,17 +7,17 @@ container_name=${4:-$1} # (Optional) The name of the container. If missing, wil docker-compose $DOCKER_COMPOSE_FILES up -d $app_name echo -e "${GREEN}Installing requirements for ${app_name}...${NC}" -docker exec -t edx.devstack.${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" +docker-compose exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" echo -e "${GREEN}Running migrations for ${app_name}...${NC}" -docker exec -t edx.devstack.${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" +docker-compose exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" echo -e "${GREEN}Creating super-user for ${app_name}...${NC}" -docker exec -t edx.devstack.${container_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" +docker-compose exec ${app_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" ./provision-ida-user.sh $app_name $client_name $client_port # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${app_name}...${NC}" -docker exec -t edx.devstack.${container_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" +docker-compose exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" diff --git a/provision-notes.sh b/provision-notes.sh index a840408741..7ae1bbada6 100755 --- a/provision-notes.sh +++ b/provision-notes.sh @@ -5,4 +5,4 @@ # This will build the elasticsearch index for notes. echo -e "${GREEN}Creating indexes for edx_notes_api...${NC}" -docker exec -t edx.devstack.edxnotesapi bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py rebuild_index --noinput' -- edx_notes_api +docker-compose exec edx_notes_api bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py rebuild_index --noinput' -- edx_notes_api diff --git a/provision-registrar.sh b/provision-registrar.sh index 3fd78f683c..c96bfb8843 100755 --- a/provision-registrar.sh +++ b/provision-registrar.sh @@ -6,17 +6,17 @@ port=18734 docker-compose $DOCKER_COMPOSE_FILES up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make requirements' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make migrate' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make createsuperuser' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make createsuperuser' -- "$name" ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c ' if ! source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make static 2>registrar_make_static.err; then echo "------- Last 100 lines of stderr"; tail regsitrar_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker-compose exec ${name} bash -c ' if ! source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make static 2>registrar_make_static.err; then echo "------- Last 100 lines of stderr"; tail regsitrar_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-retirement-user.sh b/provision-retirement-user.sh index b3a5dac2e3..3267258628 100755 --- a/provision-retirement-user.sh +++ b/provision-retirement-user.sh @@ -4,5 +4,5 @@ app_name=$1 user_name=$2 echo -e "${GREEN}Creating retirement service user ${user_name} and DOT Application ${app_name}...${NC}" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" diff --git a/provision.sh b/provision.sh index 60e6e04bfe..35b31c407f 100755 --- a/provision.sh +++ b/provision.sh @@ -122,7 +122,7 @@ fi # Ensure the MySQL server is online and usable echo "${GREEN}Waiting for MySQL.${NC}" -until docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')" &> /dev/null +until docker-compose exec -T mysql bash -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 @@ -136,20 +136,20 @@ echo -e "${GREEN}MySQL ready.${NC}" # Ensure that the MySQL databases and users are created for all IDAs. # (A no-op for databases and users that already exist). echo -e "${GREEN}Ensuring MySQL databases and users exist...${NC}" -docker exec -i edx.devstack.mysql mysql -uroot mysql < provision.sql +docker-compose exec -T mysql bash -c "mysql -uroot mysql" < provision.sql # If necessary, ensure the MongoDB server is online and usable # and create its users. if needs_mongo "$to_provision"; then echo -e "${GREEN}Waiting for MongoDB...${NC}" - until docker exec -i edx.devstack.mongo mongo --eval "printjson(db.serverStatus())" &> /dev/null + until docker-compose exec -T mongo bash -c 'mongo --eval "printjson(db.serverStatus())" &> /dev/null' do printf "." sleep 1 done echo -e "${GREEN}MongoDB ready.${NC}" echo -e "${GREEN}Creating MongoDB users...${NC}" - docker exec -i edx.devstack.mongo mongo < mongo-provision.js + docker-compose exec -T mongo bash -c 'mongo' < mongo-provision.js else echo -e "${GREEN}MongoDB preparation not required; skipping.${NC}" fi diff --git a/update-dbs-init-sql-scripts.sh b/update-dbs-init-sql-scripts.sh index 53d9830ee0..53bfefdbdb 100755 --- a/update-dbs-init-sql-scripts.sh +++ b/update-dbs-init-sql-scripts.sh @@ -24,8 +24,7 @@ make dev.clone.ssh make dev.provision.services.lms+ecommerce # dump schema and data from mysql databases in the mysql docker container and copy them to current directory in docker host -MYSQL_DOCKER_COMPOSE_SERVICE="mysql" -MYSQL_DOCKER_CONTAINER="edx.devstack.${MYSQL_DOCKER_COMPOSE_SERVICE}" +MYSQL_DOCKER_CONTAINER="$(make --silent dev.print-container.mysql)" for DB_NAME in "${DBS[@]}"; do DB_CREATION_SQL_SCRIPT="${DB_NAME}.sql" if [[ " ${EDXAPP_DBS[@]} " =~ " ${DB_NAME} " ]]; then @@ -33,6 +32,6 @@ for DB_NAME in "${DBS[@]}"; do else MYSQL_DB_USER=${ECOMMERCE_MYSQL_DB_USER} fi - docker exec ${MYSQL_DOCKER_CONTAINER} /bin/bash -c "mysqldump -u ${MYSQL_DB_USER} -p${MYSQL_DB_PASSWORD} --add-drop-database --skip-add-drop-table --databases ${DB_NAME} > ${DB_CREATION_SQL_SCRIPT}" + docker-compose exec ${MYSQL_DOCKER_CONTAINER} /bin/bash -c "mysqldump -u ${MYSQL_DB_USER} -p${MYSQL_DB_PASSWORD} --add-drop-database --skip-add-drop-table --databases ${DB_NAME} > ${DB_CREATION_SQL_SCRIPT}" docker cp ${MYSQL_DOCKER_CONTAINER}:/${DB_CREATION_SQL_SCRIPT} . done diff --git a/upgrade_mongo_3_6.sh b/upgrade_mongo_3_6.sh index 1d0ce7c9c2..c0d04ce952 100755 --- a/upgrade_mongo_3_6.sh +++ b/upgrade_mongo_3_6.sh @@ -11,11 +11,12 @@ export MONGO_VERSION=3.4.24 echo -e "${GREEN}Sarting Mongo ${MONGO_VERSION}${NC}" docker-compose up -d mongo +mongo_container="$(make -s dev.print-container.mongo)" echo -e "${GREEN}Waiting for MongoDB...${NC}" -until docker exec -i edx.devstack.mongo mongo --eval "printjson(db.serverStatus())" &> /dev/null +until docker exec "$mongo_container" bash -c 'mongo --eval "printjson(db.serverStatus())" &> /dev/null' do - if docker-compose logs mongo | grep -q "BadValue: Invalid value for version, found 3.6, expected '3.4' or '3.2'"; then + if docker logs "$mongo_container" | grep -q "BadValue: Invalid value for version, found 3.6, expected '3.4' or '3.2'"; then echo -e "${YELLOW}Already upgraded to Mongo 3.6, exiting${NC}" exit fi @@ -24,15 +25,15 @@ do done echo -e "${GREEN}MongoDB ready.${NC}" -MONGO_VERSION_LIVE=$(docker-compose exec mongo mongo --quiet --eval "printjson(db.version())") -MONGO_VERSION_COMPAT=$(docker-compose exec mongo mongo --quiet \ +MONGO_VERSION_LIVE=$(docker-compose exec -it "$mongo_container" mongo --quiet --eval "printjson(db.version())") +MONGO_VERSION_COMPAT=$(docker-compose exec -it "$mongo_container" mongo --quiet \ --eval "printjson(db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )['featureCompatibilityVersion'])") echo -e "${GREEN}Mongo Server version: ${MONGO_VERSION_LIVE}${NC}" echo -e "${GREEN}Mongo FeatureCompatibilityVersion version: ${MONGO_VERSION_COMPAT}${NC}" if echo "${MONGO_VERSION_COMPAT}" | grep -q "3\.2" ; then echo -e "${GREEN}Upgrading FeatureCompatibilityVersion to 3.4${NC}" - docker-compose exec mongo mongo --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"3.4\" } )" + docker exec -it "$mongo_container" mongo --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"3.4\" } )" else echo -e "${GREEN}FeatureCompatibilityVersion already set to 3.4${NC}" fi @@ -44,23 +45,25 @@ echo echo -e "${GREEN}Restarting Mongo on version ${MONGO_VERSION}${NC}" docker-compose up -d mongo +mongo_container="$(make -s dev.print-container.mongo)" + echo -e "${GREEN}Waiting for MongoDB...${NC}" -until docker exec -i edx.devstack.mongo mongo --eval "printjson(db.serverStatus())" &> /dev/null +until docker-compose exec mongo bash -c 'mongo --eval "printjson(db.serverStatus())" &> /dev/null' do printf "." sleep 1 done echo -e "${GREEN}MongoDB ready.${NC}" -MONGO_VERSION_LIVE=$(docker-compose exec mongo mongo --quiet --eval "printjson(db.version())") -MONGO_VERSION_COMPAT=$(docker-compose exec mongo mongo --quiet \ +MONGO_VERSION_LIVE=$(docker-compose exec -it "$mongo_container" mongo --quiet --eval "printjson(db.version())") +MONGO_VERSION_COMPAT=$(docker-compose exec -it "$mongo_container" mongo --quiet \ --eval "printjson(db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )['featureCompatibilityVersion'])") echo -e "${GREEN}Mongo Server version: ${MONGO_VERSION_LIVE}${NC}" echo -e "${GREEN}Mongo FeatureCompatibilityVersion version: ${MONGO_VERSION_COMPAT}${NC}" if echo "${MONGO_VERSION_COMPAT}" | grep -q "3\.4" ; then echo -e "${GREEN}Upgrading FeatureCompatibilityVersion to 3.6${NC}" - docker-compose exec mongo mongo --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"3.6\" } )" + docker exec -it "$mongo_container" mongo --eval "db.adminCommand( { setFeatureCompatibilityVersion: \"3.6\" } )" else echo -e "${GREEN}FeatureCompatibilityVersion already set to 3.6${NC}" fi From 09110f400d1f0ac52e6889d83552dd904b4ef5b5 Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Tue, 28 Jul 2020 11:53:21 -0400 Subject: [PATCH 2/5] Updated `README` to include necessary information when running multiple isolated devstacks for same host. Make additional adjustments to the documentation for multiple releases one machine. --- Makefile | 8 +-- README.rst | 145 +++++++++++++++++++++++++++++++++++++++++++++++++++++ options.mk | 8 ++- 3 files changed, 155 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index ff89746093..e68bdd5a42 100644 --- a/Makefile +++ b/Makefile @@ -353,16 +353,16 @@ xqueue_consumer-restart: ## Kill the XQueue development server. The watcher proc docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' lms-sass: ## Rebuild sass assets for the LMS container - docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=lms --theme-dirs /edx/app/edx-themes/edx-platform/ --themes my-theme' + docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=lms --theme-dirs /edx/app/edx-themes/edx-platform/' lms-static: ## Rebuild static assets for the LMS container - docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/ --themes my-theme' + docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/' studio-sass: ## Rebuild sass assets for the Studio container - docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=cms --theme-dirs /edx/app/edx-themes/edx-platform/ --themes my-theme' + docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=cms --theme-dirs /edx/app/edx-themes/edx-platform/' studio-static: ## Rebuild static assets for the Studio container - docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/ --themes my-theme' + docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/' static: | credentials-static discovery-static ecommerce-static lms-static studio-static ## Rebuild static assets for all service containers diff --git a/README.rst b/README.rst index d57bf7653a..426108d3c5 100644 --- a/README.rst +++ b/README.rst @@ -538,6 +538,151 @@ images until you change or unset ``OPENEDX_RELEASE`` again. To work on the master branches and ``latest`` images, unset ``OPENEDX_RELEASE`` or set it to an empty string. +How do I run multiple named Open edX releases on same machine? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +You can have multiple isolated Devstacks provisioned on a single computer now. Follow these directions to switch between the named releases. + +#. Bring down any running containers by issuing a `make stop.all`. +#. The ``COMPOSE_PROJECT_NAME`` variable is used to define Docker namespaced volumes and network based on this value, so changing it will give you a separate set of databases. This is handled for you automatically by setting the ``OPENEDX_RELEASE`` environment variable in ``options.mk`` (e.g. ``COMPOSE_PROJECT_NAME=devstack-juniper.master``. Should you want to manually override this edit the ``options.local.mk`` in the root of this repo and create the file if it does not exist. Change the devstack project name by adding the following line: + ``COMPOSE_PROJECT_NAME=`` (e.g. ``COMPOSE_PROJECT_NAME=secondarydevstack``) +#. Perform steps in `How do I run the images for a named Open edX release?`_ for specific release. +#. Follow the steps in `Getting Started`_ section to update requirements (e.g. ``make requirements``) and provision (e.g. ``make dev.provision``) the new named release containers. + +Using with OPENEDX_RELEASE: +Many of you may use the ``OPENEDX_RELEASE`` environment variable to specify the version of named release services that are checked out during Devstack provisioning. If you set ``OPENEDX_RELEASE`` and do not specify ``COMPOSE_PROJECT_NAME``, then your Devstack project name will be set as ``devstack-${OPENEDX_RELEASE}``. + +As a specific example, if ``OPENEDX_RELEASE`` is set in your environment as ``juniper.master``, then ``COMPOSE_PROJECT_NAME`` will default to ``devstack-juniper.master`` instead of ``devstack``. + +The implication of this is that you can switch between isolated Devstack databases by changing the value of the ``OPENEDX_RELEASE`` environment variable. + +Switch between your Devstack projects by doing the following: + +#. Bring down the containers by issuing a `make stop.all`. +#. Edit the project name in ``options.local.mk`` or set ``OPENEDX_RELEASE`` and let the ``COMPOSE_PROJECT_NAME`` be assigned automatically. +#. Bring up the containers by issuing a ``make dev.up`` or ``make dev.up.all``. + +Examples of Docker Service Names After Setting the ``COMPOSE_PROJECT_NAME`` variable. Notice that the **devstack-juniper.master** name represents the ``COMPOSE_PROJECT_NAME``. + +- edx.devstack-juniper.master.lms +- edx.devstack-juniper.master.mysql + +Each instance has an isolated set of databases. This could, for example, be used to quickly switch between versions of Open edX without hitting as many issues with migrations, data integrity, etc. + +Unfortunately, this **does not** currently support running Devstacks simultaneously, because we hard-code host port numbers all over the place, and two running containers cannot share the same host port. + +Questions & Troubleshooting – Multiple Named Open edX Releases on Same Machine +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This broke my existing Devstack! +******************************** +Try posting on Discourse or Slack to see if you have the same issue as any others. If you think you have found a bug, file a CR ticket. + +I’m getting errors related to ports already being used. +******************************************************* +Make sure you bring down your devstack before changing the value of COMPOSE_PROJECT_NAME. If you forgot to, change the COMPOSE_PROJECT_NAME back to its original value, run ``make dev.down``, and then try again. + +I have custom scripts/compose files that integrate with or extend Devstack. Will those still work? +************************************************************************************************** +With the default value of COMPOSE_PROJECT_NAME = devstack, they should still work. If you choose a different COMPOSE_PROJECT_NAME, your extensions will likely break, because the names of containers change along with the project name. + +How do I enable environment variables for current directory using 'direnv'? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Recommend separating the named releases into different directories since `direenv` unloads/loads different environment variables per directory. + +#. Install `direnv` using instructions on https://direnv.net/. Below you will find additional setup at the time of this writing so refer to latest of `direnv` site for additional configuration needed. + +#. Setup the following configuration to hook `direnv` for local directory environment overrides. There are two examples for BASH or ZSH (Mac OS X) shells. + + .. code:: sh + + ## ~/.bashrc for BASH shell + + ## Hook in `direnv` for local directory environment overrides. + ## https://direnv.net/docs/hook.html + eval "$(direnv hook bash)" + + # https://github.com/direnv/direnv/wiki/Python#bash + show_virtual_env() { + if [[ -n "$VIRTUAL_ENV" && -n "$DIRENV_DIR" ]]; then + echo "($(basename $VIRTUAL_ENV))" + fi + } + export -f show_virtual_env + PS1='$(show_virtual_env)'$PS1 + + # --------------------------------------------------- + + ## ~/.zshrc for ZSH shell for Mac OS X. + + ## Hook in `direnv` for local directory environment setup. + ## https://direnv.net/docs/hook.html + eval "$(direnv hook zsh)" + + # https://github.com/direnv/direnv/wiki/Python#zsh + setopt PROMPT_SUBST + + show_virtual_env() { + if [[ -n "$VIRTUAL_ENV" && -n "$DIRENV_DIR" ]]; then + echo "($(basename $VIRTUAL_ENV))" + fi + } + PS1='$(show_virtual_env)'$PS1 + +#. Setup `layout_python-venv` function to be used in local project directory `.envrc` file. + + .. code:: sh + + ## ~/.config/direnv/direnvrc + + # https://github.com/direnv/direnv/wiki/Python#venv-stdlib-module + + realpath() { + [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}" + } + layout_python-venv() { + local python=${1:-python3} + [[ $# -gt 0 ]] && shift + unset PYTHONHOME + if [[ -n $VIRTUAL_ENV ]]; then + VIRTUAL_ENV=$(realpath "${VIRTUAL_ENV}") + else + local python_version + python_version=$("$python" -c "import platform; print(platform.python_version())") + if [[ -z $python_version ]]; then + log_error "Could not detect Python version" + return 1 + fi + VIRTUAL_ENV=$PWD/.direnv/python-venv-$python_version + fi + export VIRTUAL_ENV + if [[ ! -d $VIRTUAL_ENV ]]; then + log_status "no venv found; creating $VIRTUAL_ENV" + "$python" -m venv "$VIRTUAL_ENV" + fi + + PATH="${VIRTUAL_ENV}/bin:${PATH}" + export PATH + } + +#. Example `.envrc` file used in project directory. Need to make sure that each release root has this unique file. + + .. code:: sh + + # Open edX named release project directory root. + ## /devstack.juniper/.envrc + + # https://discuss.openedx.org/t/docker-devstack-multiple-releases-one-machine/1902/10 + + # This is handled when OPENEDX_RELEASE is set. Leaving this in for manual override. + # export COMPOSE_PROJECT_NAME=devstack-juniper + + export DEVSTACK_WORKSPACE="$(pwd)" + export OPENEDX_RELEASE=juniper.master + export VIRTUAL_ENV="$(pwd)/devstack/venv" + + # https://github.com/direnv/direnv/wiki/Python#virtualenv + layout python-venv + How do I create relational database dumps? ------------------------------------------ We use relational database dumps to spend less time running relational database diff --git a/options.mk b/options.mk index 0e0a456099..62a16f0afc 100644 --- a/options.mk +++ b/options.mk @@ -9,8 +9,12 @@ DEVSTACK_WORKSPACE ?= $(shell pwd)/.. # Name of Docker Compose project. -# See https://docs.docker.com/compose/reference/envvars/#compose_project_name -# Defaults to 'devstack' should OPENEDX_RELEASE not be defined. +# Volumes and network are namespaced based on this value, +# so changing it will give you a separate set of databases. +# See https://docs.docker.com/compose/reference/envvars/#compose_project_name +# If OPENEDX_RELAESE is defined, defaults to `devstack-${OPENEDX_RELEASE}`; +# otherwise, it defaults to `devstack`. +# Be sure to bring down services before changing the value of `COMPOSE_PROJECT_NAME`. ifdef OPENEDX_RELEASE COMPOSE_PROJECT_NAME ?= devstack-${OPENEDX_RELEASE} else From 357f8a8360610885890a10eb0b3b9e77e02040a2 Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Wed, 29 Jul 2020 19:37:07 -0400 Subject: [PATCH 3/5] Make sure to pass in `DOCKER_COMPOSE_FILES` to `docker-compose exec` commands for services outside of `docker-compose.yml`. Makes sure that other services (e.g. `xqueue`, etc) that are not defined in the docker-compose.yml file have the correct configuration passed to the `docker-compose exec` command to work properly. --- Makefile | 66 +++++++++++++++--------------- Makefile.edx | 4 +- course-generator/create-courses.sh | 12 +++--- enterprise/provision.sh | 4 +- marketing.mk | 2 +- programs/provision.sh | 2 +- provision-analyticspipeline.sh | 12 +++--- provision-credentials.sh | 10 ++--- provision-discovery.sh | 8 ++-- provision-e2e.sh | 8 ++-- provision-ecommerce.sh | 6 +-- provision-forum.sh | 2 +- provision-ida-user.sh | 6 +-- provision-ida.sh | 8 ++-- provision-lms.sh | 16 ++++---- provision-notes.sh | 2 +- provision-registrar.sh | 8 ++-- provision-retirement-user.sh | 4 +- provision.sh | 8 ++-- update-dbs-init-sql-scripts.sh | 2 +- upgrade_mongo_3_6.sh | 10 ++--- 21 files changed, 100 insertions(+), 100 deletions(-) diff --git a/Makefile b/Makefile index e68bdd5a42..41ebf3a72d 100644 --- a/Makefile +++ b/Makefile @@ -179,7 +179,7 @@ dev.up.all: dev.up dev.up.watchers ## Bring up all services with host volumes, i dev.print-container.%: ## Get the ID of the running container for a given service. @# Can be run as ``make --silent dev.print-container.$service`` for just ID. - @echo $$(docker-compose ps --quiet $*) + @echo $$(docker-compose $(DOCKER_COMPOSE_FILES) ps --quiet $*) # TODO: Improve or rip out Docker Sync targets. # They are not well-fleshed-out and it is not clear if anyone uses them. @@ -206,7 +206,7 @@ provision: | dev.provision ## This command will be deprecated in a future releas stop: ## Stop all services (test -d .docker-sync && docker-sync stop) || true ## Ignore failure here - docker-compose stop + docker-compose $(DOCKER_COMPOSE_FILES) stop stop.watchers: ## Stop asset watchers docker-compose $(DOCKER_COMPOSE_FILES) stop lms_watcher studio_watcher @@ -282,45 +282,45 @@ restore: ## Restore all data volumes from the host. WARNING: THIS WILL OVERWRIT # TODO: Print out help for this target. Even better if we can iterate over the # services in docker-compose.yml, and print the actual service names. %-shell: ## Run a shell on the specified service container - docker-compose exec $* /bin/bash + docker-compose $(DOCKER_COMPOSE_FILES) exec $* /bin/bash analyticspipeline-shell: ## Run a shell on the analytics pipeline container - docker-compose exec analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open credentials-shell: ## Run a shell on the credentials container - docker-compose exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' + docker-compose $(DOCKER_COMPOSE_FILES) exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' discovery-shell: ## Run a shell on the discovery container - docker-compose exec discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open ecommerce-shell: ## Run a shell on the ecommerce container - docker-compose exec ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open e2e-shell: ## Start the end-to-end tests container with a shell docker-compose run --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash registrar-shell: ## Run a shell on the registrar site container - docker-compose exec registrar env TERM=$(TERM) /edx/app/registrar/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec registrar env TERM=$(TERM) /edx/app/registrar/devstack.sh open %-update-db: ## Run migrations for the specified service container - docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' + docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' studio-update-db: ## Run migrations for the Studio container - docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' + docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' lms-update-db: ## Run migrations LMS container - docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' + docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' update-db: | $(DB_MIGRATION_TARGETS) ## Run the migrations for DEFAULT_SERVICES forum-restart-devserver: ## Kill the forum's Sinatra development server. The watcher process will restart it. - docker-compose forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose $(DOCKER_COMPOSE_FILES) forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")' lms-shell: ## Run a shell on the LMS container - docker-compose exec lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open lms-watcher-shell: ## Run a shell on the LMS watcher container - docker-compose exec lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open %-attach: ## Attach to the specified service container process to use the debugger & see logs. docker attach "$$(make --silent dev.print-container.$*)" @@ -328,41 +328,41 @@ lms-watcher-shell: ## Run a shell on the LMS watcher container lms-restart: lms-restart-devserver studio-shell: ## Run a shell on the Studio container - docker-compose exec studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-watcher-shell: ## Run a shell on the studio watcher container - docker-compose exec studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-restart: studio-restart-devserver xqueue-shell: ## Run a shell on the XQueue container - docker-compose exec xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue-restart: xqueue-restart-devserver %-restart-devserver: ## Kill a service's Django development server. The watcher process should restart it. - docker-compose exec $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' xqueue_consumer-shell: ## Run a shell on the XQueue consumer container - docker-compose exec xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue_consumer-restart: ## Kill the XQueue development server. The watcher process will restart it. - docker-compose exec xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")' %-static: ## Rebuild static assets for the specified service container - docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' + docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' lms-sass: ## Rebuild sass assets for the LMS container - docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=lms --theme-dirs /edx/app/edx-themes/edx-platform/' + docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=lms --theme-dirs /edx/app/edx-themes/edx-platform/' lms-static: ## Rebuild static assets for the LMS container - docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/' + docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/' studio-sass: ## Rebuild sass assets for the Studio container - docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=cms --theme-dirs /edx/app/edx-themes/edx-platform/' + docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=cms --theme-dirs /edx/app/edx-themes/edx-platform/' studio-static: ## Rebuild static assets for the Studio container - docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/' + docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/' static: | credentials-static discovery-static ecommerce-static lms-static studio-static ## Rebuild static assets for all service containers @@ -375,24 +375,24 @@ e2e-tests: ## Run the end-to-end tests against the service containers validate-lms-volume: ## Validate that changes to the local workspace are reflected in the LMS container touch $(DEVSTACK_WORKSPACE)/edx-platform/testfile - docker-compose exec -T lms ls /edx/app/edxapp/edx-platform/testfile + docker-compose $(DOCKER_COMPOSE_FILES) exec -T lms ls /edx/app/edxapp/edx-platform/testfile rm $(DEVSTACK_WORKSPACE)/edx-platform/testfile vnc-passwords: ## Get the VNC passwords for the Chrome and Firefox Selenium containers - @docker-compose logs chrome 2>&1 | grep "VNC password" | tail -1 - @docker-compose logs firefox 2>&1 | grep "VNC password" | tail -1 + @docker-compose $(DOCKER_COMPOSE_FILES) logs chrome 2>&1 | grep "VNC password" | tail -1 + @docker-compose $(DOCKER_COMPOSE_FILES) logs firefox 2>&1 | grep "VNC password" | tail -1 devpi-password: ## Get the root devpi password for the devpi container - docker-compose exec devpi bash -c "cat /data/server/.serverpassword" + docker-compose $(DOCKER_COMPOSE_FILES) exec devpi bash -c "cat /data/server/.serverpassword" mysql-shell: ## Run a shell on the mysql container - docker-compose exec mysql bash + docker-compose $(DOCKER_COMPOSE_FILES) exec mysql bash mysql-shell-edxapp: ## Run a mysql shell on the edxapp database - docker-compose exec mysql bash -c "mysql edxapp" + docker-compose $(DOCKER_COMPOSE_FILES) exec mysql bash -c "mysql edxapp" mongo-shell: ## Run a shell on the mongo container - docker-compose exec mongo bash + docker-compose $(DOCKER_COMPOSE_FILES) exec mongo bash dev.provision.analytics_pipeline: dev.provision.services.analyticspipeline @@ -411,7 +411,7 @@ stop.analytics_pipeline: ## Stop all Analytics pipeline services. sparkworker vertica analyticspipeline hadoop-application-logs-%: ## View hadoop logs by application Id - docker-compose exec analytics_pipeline.nodemanager yarn logs -applicationId $* + docker-compose $(DOCKER_COMPOSE_FILES) exec analytics_pipeline.nodemanager yarn logs -applicationId $* # Provisions studio, ecommerce, and marketing with course(s) in test-course.json # Modify test-course.json before running this make target to generate a custom course diff --git a/Makefile.edx b/Makefile.edx index 030273960e..3a32bbe319 100644 --- a/Makefile.edx +++ b/Makefile.edx @@ -25,10 +25,10 @@ dev.provision.whitelabel: whitelabel-tests: docker-compose run -T -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh $(make --silent dev.print-container.devstack.whitelabel)":/tmp/run_whitelabel_tests.sh - docker-compose exec devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh + docker-compose $(DOCKER_COMPOSE_FILES) exec devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh whitelabel-cleanup: docker-compose rm --force devstack.whitelabel whitelabel-shell: ## Start a whitelabel test shell session - docker-compose exec -it devstack.whitelabel env TERM=$(TERM) bash + docker-compose $(DOCKER_COMPOSE_FILES) exec -it devstack.whitelabel env TERM=$(TERM) bash diff --git a/course-generator/create-courses.sh b/course-generator/create-courses.sh index b7ee059a9e..54e2bc3629 100755 --- a/course-generator/create-courses.sh +++ b/course-generator/create-courses.sh @@ -8,21 +8,21 @@ echo "Parsing options" container_error=false for arg in "$@"; do if [ $arg == "--studio" ]; then - if [ ! "$(docker-compose exec lms bash -c 'echo "Course will be created for studio"; exit $?')" ]; then + if [ ! "$(docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'echo "Course will be created for studio"; exit $?')" ]; then echo "Issue with studio container" container_error=true else studio=true fi elif [ $arg == "--ecommerce" ]; then - if [ ! "$(docker-compose exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then + if [ ! "$(docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then echo "Issue with ecommerce container" container_error=true else ecommerce=true fi elif [ $arg == "--marketing" ]; then - if [ ! "$(docker-compose exec marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then + if [ ! "$(docker-compose $DOCKER_COMPOSE_FILES exec marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then echo "Issue with marketing container. Course creation will proceed without marketing container." else marketing=true @@ -49,15 +49,15 @@ done < "${@: -1}" if $studio ; then echo "Creating courses on studio." - docker-compose exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'" + docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'" fi if $ecommerce ; then echo "Creating courses on ecommerce." - docker-compose exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" + docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" fi if $marketing ; then echo "Creating courses on marketing." - docker-compose exec marketing bash -c "drush generate_courses '$course_json'" + docker-compose $DOCKER_COMPOSE_FILES exec marketing bash -c "drush generate_courses '$course_json'" fi diff --git a/enterprise/provision.sh b/enterprise/provision.sh index 559e02d671..cfb5676fd1 100755 --- a/enterprise/provision.sh +++ b/enterprise/provision.sh @@ -1,2 +1,2 @@ -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user enterprise_worker enterprise_worker@example.com --staff' -cat enterprise/worker_permissions.py | docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user enterprise_worker enterprise_worker@example.com --staff' +cat enterprise/worker_permissions.py | docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' diff --git a/marketing.mk b/marketing.mk index c1496c3ef0..a8e1b4b59f 100644 --- a/marketing.mk +++ b/marketing.mk @@ -3,7 +3,7 @@ help-marketing: ## Display this help message @awk -F ':.*?## ' '/^[a-zA-Z]/ && NF==2 {printf "\033[36m %-25s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | grep marketing | sort marketing-shell: ## Run a shell on the marketing site container - docker-compose exec marketing env TERM=$(TERM) bash -c 'cd /edx/app/edx-mktg/edx-mktg; exec /bin/bash -sh' + docker-compose $(DOCKER_COMPOSE_FILES) exec marketing env TERM=$(TERM) bash -c 'cd /edx/app/edx-mktg/edx-mktg; exec /bin/bash -sh' stop-marketing: ## Stop all services (including the marketing site) with host volumes docker-compose -f docker-compose.yml -f docker-compose-host.yml -f docker-compose-themes.yml -f docker-compose-marketing-site.yml -f docker-compose-marketing-site-host.yml stop diff --git a/programs/provision.sh b/programs/provision.sh index 967ae39774..3949babfe4 100755 --- a/programs/provision.sh +++ b/programs/provision.sh @@ -44,7 +44,7 @@ docker_exec() { /edx/app/$app/$repo/manage.py $cmd " - docker-compose exec -T "$service" bash -c "$CMDS" + docker-compose $DOCKER_COMPOSE_FILES exec -T "$service" bash -c "$CMDS" } provision_ida() { diff --git a/provision-analyticspipeline.sh b/provision-analyticspipeline.sh index 0fd4d7c81a..ffbe7e7176 100755 --- a/provision-analyticspipeline.sh +++ b/provision-analyticspipeline.sh @@ -22,24 +22,24 @@ docker-compose $DOCKER_COMPOSE_FILES up -d analyticspipeline # Analytics pipeline has dependency on lms but we only need its db schema & not full lms. So we'll just load their db # schemas as part of analytics pipeline provisioning. If there is a need of a fully fledged LMS, then provision lms # by following their documentation. -if [[ ! -z "`docker-compose exec mysql mysql -uroot -se "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='edxapp'" 2>&1`" ]]; +if [[ ! -z "`docker-compose $DOCKER_COMPOSE_FILES exec mysql mysql -uroot -se "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='edxapp'" 2>&1`" ]]; then echo -e "${GREEN}LMS DB exists, skipping lms schema load.${NC}" else echo -e "${GREEN}LMS DB not found, provisioning lms schema.${NC}" - docker-compose exec mysql bash -c 'mysql -uroot mysql' < provision.sql + docker-compose $DOCKER_COMPOSE_FILES exec mysql bash -c 'mysql -uroot mysql' < provision.sql ./load-db.sh edxapp docker-compose $DOCKER_COMPOSE_FILES up -d lms studio - docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' + docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' #Installing prereqs crashes the process docker-compose restart lms # Run edxapp migrations first since they are needed for the service users and OAuth clients - docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' + docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' fi echo -e "${GREEN}LMS database provisioned successfully...${NC}" echo -e "${GREEN}Creating databases and users...${NC}" -docker-compose exec mysql bash -c 'mysql -uroot mysql' < provision-analytics-pipeline.sql +docker-compose $DOCKER_COMPOSE_FILES exec mysql bash -c 'mysql -uroot mysql' < provision-analytics-pipeline.sql # initialize hive metastore echo -e "${GREEN}Initializing HIVE metastore...${NC}" @@ -56,4 +56,4 @@ done sleep 10 # for datanode & other services to activate echo -e "${GREEN}Namenode is ready!${NC}" -docker-compose exec -u hadoop analytics_pipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' +docker-compose $DOCKER_COMPOSE_FILES exec -u hadoop analytics_pipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' diff --git a/provision-credentials.sh b/provision-credentials.sh index 0334f2966b..1029ce339d 100755 --- a/provision-credentials.sh +++ b/provision-credentials.sh @@ -9,20 +9,20 @@ port=18150 docker-compose $DOCKER_COMPOSE_FILES up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" echo -e "${GREEN}Configuring site for ${name}...${NC}" -docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker-compose exec ${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-discovery.sh b/provision-discovery.sh index 9b72e356df..d58f79351e 100755 --- a/provision-discovery.sh +++ b/provision-discovery.sh @@ -1,10 +1,10 @@ # Provisioning script for the discovery service ./provision-ida.sh discovery discovery 18381 -docker-compose exec -T discovery bash -c 'rm -rf /edx/var/discovery/*' -docker-compose exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 1 --site-domain localhost:18381 --code edx --name edX --courses-api-url "http://edx.devstack.lms:18000/api/courses/v1/" --ecommerce-api-url "http://edx.devstack.ecommerce:18130/api/v2/" --organizations-api-url "http://edx.devstack.lms:18000/api/organizations/v0/" --lms-url "http://edx.devstack.lms:18000/"' -docker-compose exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py refresh_course_metadata' -docker-compose exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py update_index --disable-change-limit' +docker-compose $DOCKER_COMPOSE_FILES exec -T discovery bash -c 'rm -rf /edx/var/discovery/*' +docker-compose $DOCKER_COMPOSE_FILES exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 1 --site-domain localhost:18381 --code edx --name edX --courses-api-url "http://edx.devstack.lms:18000/api/courses/v1/" --ecommerce-api-url "http://edx.devstack.ecommerce:18130/api/v2/" --organizations-api-url "http://edx.devstack.lms:18000/api/organizations/v0/" --lms-url "http://edx.devstack.lms:18000/"' +docker-compose $DOCKER_COMPOSE_FILES exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py refresh_course_metadata' +docker-compose $DOCKER_COMPOSE_FILES exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py update_index --disable-change-limit' # Add demo program ./programs/provision.sh discovery diff --git a/provision-e2e.sh b/provision-e2e.sh index 86722225d1..baee623cd0 100755 --- a/provision-e2e.sh +++ b/provision-e2e.sh @@ -13,11 +13,11 @@ fi docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz "$(make --silent dev.print-container.studio)":/tmp/ # Extract the test course tarball -docker-compose exec -T studio bash -c 'cd /tmp && tar xzf course.tar.gz' +docker-compose $DOCKER_COMPOSE_FILES exec -T studio bash -c 'cd /tmp && tar xzf course.tar.gz' # Import the course content -docker-compose exec -T studio bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker import /tmp course' +docker-compose $DOCKER_COMPOSE_FILES exec -T studio bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker import /tmp course' # Clean up the temp files -docker-compose exec -T studio bash -c 'rm /tmp/course.tar.gz' -docker-compose exec -T studio bash -c 'rm -r /tmp/course' +docker-compose $DOCKER_COMPOSE_FILES exec -T studio bash -c 'rm /tmp/course.tar.gz' +docker-compose $DOCKER_COMPOSE_FILES exec -T studio bash -c 'rm -r /tmp/course' diff --git a/provision-ecommerce.sh b/provision-ecommerce.sh index 00068b6c0e..3846ad7045 100755 --- a/provision-ecommerce.sh +++ b/provision-ecommerce.sh @@ -4,6 +4,6 @@ ./provision-ida.sh ecommerce ecommerce 18130 # Configure ecommerce -docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --client-id=ecommerce-key --client-secret=ecommerce-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --journals_api_url=http://journals.app:18606/api/v1/' -docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' -docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' +docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --client-id=ecommerce-key --client-secret=ecommerce-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --journals_api_url=http://journals.app:18606/api/v1/' +docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' +docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' diff --git a/provision-forum.sh b/provision-forum.sh index 6196ac0176..8e1f13b864 100755 --- a/provision-forum.sh +++ b/provision-forum.sh @@ -3,4 +3,4 @@ set -o pipefail set -x docker-compose $DOCKER_COMPOSE_FILES up -d forum -docker-compose exec -T forum bash -c 'source /edx/app/forum/ruby_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/' +docker-compose $DOCKER_COMPOSE_FILES exec -T forum bash -c 'source /edx/app/forum/ruby_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/' diff --git a/provision-ida-user.sh b/provision-ida-user.sh index 965de8a566..38e11e0a5a 100755 --- a/provision-ida-user.sh +++ b/provision-ida-user.sh @@ -7,8 +7,8 @@ client_port=$3 echo -e "${GREEN}Creating service user and OAuth2 applications for ${app_name}...${NC}" # Create the service user. -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" # Create the DOT applications - one for single sign-on and one for backend service IDA-to-IDA authentication. -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "http://localhost:$3/complete/edx-oauth2/" --client-id "$1-sso-key" --client-secret "$1-sso-secret" --scopes "user_id" $1-sso $1_worker' -- "$app_name" "$client_name" "$client_port" -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "$1-backend-service-key" --client-secret "$1-backend-service-secret" $1-backend-service $1_worker' -- "$app_name" "$client_name" "$client_port" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "http://localhost:$3/complete/edx-oauth2/" --client-id "$1-sso-key" --client-secret "$1-sso-secret" --scopes "user_id" $1-sso $1_worker' -- "$app_name" "$client_name" "$client_port" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "$1-backend-service-key" --client-secret "$1-backend-service-secret" $1-backend-service $1_worker' -- "$app_name" "$client_name" "$client_port" diff --git a/provision-ida.sh b/provision-ida.sh index b1f967d121..e93282f28a 100755 --- a/provision-ida.sh +++ b/provision-ida.sh @@ -7,17 +7,17 @@ container_name=${4:-$1} # (Optional) The name of the container. If missing, wil docker-compose $DOCKER_COMPOSE_FILES up -d $app_name echo -e "${GREEN}Installing requirements for ${app_name}...${NC}" -docker-compose exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" echo -e "${GREEN}Running migrations for ${app_name}...${NC}" -docker-compose exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" echo -e "${GREEN}Creating super-user for ${app_name}...${NC}" -docker-compose exec ${app_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec ${app_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" ./provision-ida-user.sh $app_name $client_name $client_port # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${app_name}...${NC}" -docker-compose exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" diff --git a/provision-lms.sh b/provision-lms.sh index 7ef1b8e252..6c69176741 100755 --- a/provision-lms.sh +++ b/provision-lms.sh @@ -13,33 +13,33 @@ for app in "${apps[@]}"; do docker-compose $DOCKER_COMPOSE_FILES up -d $app done -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' #Installing prereqs crashes the process docker-compose restart lms # Run edxapp migrations first since they are needed for the service users and OAuth clients -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' # Create a superuser for edxapp -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user edx edx@example.com --superuser --staff' -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username=\"edx\"); user.set_password(\"edx\"); user.save()" | python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user edx edx@example.com --superuser --staff' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username=\"edx\"); user.set_password(\"edx\"); user.save()" | python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' # Create an enterprise service user for edxapp and give them appropriate permissions ./enterprise/provision.sh # Enable the LMS-E-Commerce integration -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce' # Create demo course and users -docker-compose exec -T lms bash -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"' # Fix missing vendor file by clearing the cache -docker-compose exec -T lms bash -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1' # Create static assets for both LMS and Studio for app in "${apps[@]}"; do - docker-compose exec -T $app bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker' + docker-compose $DOCKER_COMPOSE_FILES exec -T $app bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker' done # Provision a retirement service account user diff --git a/provision-notes.sh b/provision-notes.sh index 7ae1bbada6..4341580dfd 100755 --- a/provision-notes.sh +++ b/provision-notes.sh @@ -5,4 +5,4 @@ # This will build the elasticsearch index for notes. echo -e "${GREEN}Creating indexes for edx_notes_api...${NC}" -docker-compose exec edx_notes_api bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py rebuild_index --noinput' -- edx_notes_api +docker-compose $DOCKER_COMPOSE_FILES exec edx_notes_api bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py rebuild_index --noinput' -- edx_notes_api diff --git a/provision-registrar.sh b/provision-registrar.sh index c96bfb8843..e2b30e1ebf 100755 --- a/provision-registrar.sh +++ b/provision-registrar.sh @@ -6,17 +6,17 @@ port=18734 docker-compose $DOCKER_COMPOSE_FILES up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker-compose exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make requirements' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker-compose exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make migrate' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker-compose exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make createsuperuser' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make createsuperuser' -- "$name" ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker-compose exec ${name} bash -c ' if ! source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make static 2>registrar_make_static.err; then echo "------- Last 100 lines of stderr"; tail regsitrar_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c ' if ! source /edx/app/registrar/registrar_env && cd /edx/app/registrar/registrar && make static 2>registrar_make_static.err; then echo "------- Last 100 lines of stderr"; tail regsitrar_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-retirement-user.sh b/provision-retirement-user.sh index 3267258628..9a9f209c6c 100755 --- a/provision-retirement-user.sh +++ b/provision-retirement-user.sh @@ -4,5 +4,5 @@ app_name=$1 user_name=$2 echo -e "${GREEN}Creating retirement service user ${user_name} and DOT Application ${app_name}...${NC}" -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" diff --git a/provision.sh b/provision.sh index 35b31c407f..220be49111 100755 --- a/provision.sh +++ b/provision.sh @@ -122,7 +122,7 @@ fi # Ensure the MySQL server is online and usable echo "${GREEN}Waiting for MySQL.${NC}" -until docker-compose exec -T mysql bash -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null +until docker-compose $DOCKER_COMPOSE_FILES exec -T mysql bash -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 @@ -136,20 +136,20 @@ echo -e "${GREEN}MySQL ready.${NC}" # Ensure that the MySQL databases and users are created for all IDAs. # (A no-op for databases and users that already exist). echo -e "${GREEN}Ensuring MySQL databases and users exist...${NC}" -docker-compose exec -T mysql bash -c "mysql -uroot mysql" < provision.sql +docker-compose $DOCKER_COMPOSE_FILES exec -T mysql bash -c "mysql -uroot mysql" < provision.sql # If necessary, ensure the MongoDB server is online and usable # and create its users. if needs_mongo "$to_provision"; then echo -e "${GREEN}Waiting for MongoDB...${NC}" - until docker-compose exec -T mongo bash -c 'mongo --eval "printjson(db.serverStatus())" &> /dev/null' + until docker-compose $DOCKER_COMPOSE_FILES exec -T mongo bash -c 'mongo --eval "printjson(db.serverStatus())" &> /dev/null' do printf "." sleep 1 done echo -e "${GREEN}MongoDB ready.${NC}" echo -e "${GREEN}Creating MongoDB users...${NC}" - docker-compose exec -T mongo bash -c 'mongo' < mongo-provision.js + docker-compose $DOCKER_COMPOSE_FILES exec -T mongo bash -c 'mongo' < mongo-provision.js else echo -e "${GREEN}MongoDB preparation not required; skipping.${NC}" fi diff --git a/update-dbs-init-sql-scripts.sh b/update-dbs-init-sql-scripts.sh index 53bfefdbdb..591144434b 100755 --- a/update-dbs-init-sql-scripts.sh +++ b/update-dbs-init-sql-scripts.sh @@ -32,6 +32,6 @@ for DB_NAME in "${DBS[@]}"; do else MYSQL_DB_USER=${ECOMMERCE_MYSQL_DB_USER} fi - docker-compose exec ${MYSQL_DOCKER_CONTAINER} /bin/bash -c "mysqldump -u ${MYSQL_DB_USER} -p${MYSQL_DB_PASSWORD} --add-drop-database --skip-add-drop-table --databases ${DB_NAME} > ${DB_CREATION_SQL_SCRIPT}" + docker-compose $DOCKER_COMPOSE_FILES exec ${MYSQL_DOCKER_CONTAINER} /bin/bash -c "mysqldump -u ${MYSQL_DB_USER} -p${MYSQL_DB_PASSWORD} --add-drop-database --skip-add-drop-table --databases ${DB_NAME} > ${DB_CREATION_SQL_SCRIPT}" docker cp ${MYSQL_DOCKER_CONTAINER}:/${DB_CREATION_SQL_SCRIPT} . done diff --git a/upgrade_mongo_3_6.sh b/upgrade_mongo_3_6.sh index c0d04ce952..28ec53480e 100755 --- a/upgrade_mongo_3_6.sh +++ b/upgrade_mongo_3_6.sh @@ -25,8 +25,8 @@ do done echo -e "${GREEN}MongoDB ready.${NC}" -MONGO_VERSION_LIVE=$(docker-compose exec -it "$mongo_container" mongo --quiet --eval "printjson(db.version())") -MONGO_VERSION_COMPAT=$(docker-compose exec -it "$mongo_container" mongo --quiet \ +MONGO_VERSION_LIVE=$(docker-compose $DOCKER_COMPOSE_FILES exec -it "$mongo_container" mongo --quiet --eval "printjson(db.version())") +MONGO_VERSION_COMPAT=$(docker-compose $DOCKER_COMPOSE_FILES exec -it "$mongo_container" mongo --quiet \ --eval "printjson(db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )['featureCompatibilityVersion'])") echo -e "${GREEN}Mongo Server version: ${MONGO_VERSION_LIVE}${NC}" echo -e "${GREEN}Mongo FeatureCompatibilityVersion version: ${MONGO_VERSION_COMPAT}${NC}" @@ -48,15 +48,15 @@ docker-compose up -d mongo mongo_container="$(make -s dev.print-container.mongo)" echo -e "${GREEN}Waiting for MongoDB...${NC}" -until docker-compose exec mongo bash -c 'mongo --eval "printjson(db.serverStatus())" &> /dev/null' +until docker-compose $DOCKER_COMPOSE_FILES exec mongo bash -c 'mongo --eval "printjson(db.serverStatus())" &> /dev/null' do printf "." sleep 1 done echo -e "${GREEN}MongoDB ready.${NC}" -MONGO_VERSION_LIVE=$(docker-compose exec -it "$mongo_container" mongo --quiet --eval "printjson(db.version())") -MONGO_VERSION_COMPAT=$(docker-compose exec -it "$mongo_container" mongo --quiet \ +MONGO_VERSION_LIVE=$(docker-compose $DOCKER_COMPOSE_FILES exec -it "$mongo_container" mongo --quiet --eval "printjson(db.version())") +MONGO_VERSION_COMPAT=$(docker-compose $DOCKER_COMPOSE_FILES exec -it "$mongo_container" mongo --quiet \ --eval "printjson(db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )['featureCompatibilityVersion'])") echo -e "${GREEN}Mongo Server version: ${MONGO_VERSION_LIVE}${NC}" echo -e "${GREEN}Mongo FeatureCompatibilityVersion version: ${MONGO_VERSION_COMPAT}${NC}" From ffcafdfa865ca94bbf57bd801af7829ac8fe3781 Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Mon, 10 Aug 2020 16:32:08 -0400 Subject: [PATCH 4/5] Add `xqueue` to default services to provision initially. --- options.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options.mk b/options.mk index 62a16f0afc..b09035515b 100644 --- a/options.mk +++ b/options.mk @@ -47,7 +47,7 @@ FS_SYNC_STRATEGY ?= local-mounts # TODO: Re-evaluate this list and consider paring it down to a tighter core. # The current value was chosen such that it would not change the existing # Devstack behavior. -DEFAULT_SERVICES ?= lms+studio+ecommerce+discovery+credentials+forum+edx_notes_api+gradebook+frontend-app-publisher +DEFAULT_SERVICES ?= lms+studio+ecommerce+discovery+xqueue+credentials+forum+edx_notes_api+gradebook+frontend-app-publisher # List of all services with database migrations. # Services must provide a Makefile target named: $(service)-update-db From 35d197b47d5d42efba6372812ab3fc55b26782e1 Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Mon, 17 Aug 2020 08:41:56 -0400 Subject: [PATCH 5/5] Make additional adjustments to the documentation for multiple releases one machine (regisb review). Comments came from `master` https://github.com/edx/devstack/pull/569/commits/c0b482d9ff1355b3d59ee25b63b3ca79d93ff372 Fix issue with `docker-compose config` not outputting configuration information when `$(DOCKER_COMPOSE_FILES)` was excluded from the command. It appears that leaving out `$(DOCKER_COMPOSE_FILES)` caused the `docker-compose config` to not include the **volume** information for services within the `docker-compose.yml` file, as well as, excluding additional services like `docker-compose-{analytics-pipeline,watchers}.yml` to name a few. Update provision-ecommerce.sh with recommended changes from @morenol Co-authored-by: morenol <13-10934@usb.ve> Merging additional updates based on #569 changes for the `readme` file. - All changes from https://github.com/edx/devstack/pull/569 changes from this. --- Makefile | 2 +- README.rst | 62 ++++++++++++++++++++++++++++++------------ provision-ecommerce.sh | 2 +- 3 files changed, 46 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 41ebf3a72d..6e2bf4ac77 100644 --- a/Makefile +++ b/Makefile @@ -267,7 +267,7 @@ pull: dev.pull pull.xqueue: dev.pull.without-deps.xqueue+xqueue_consumer validate: ## Validate the devstack configuration - docker-compose config + docker-compose $(DOCKER_COMPOSE_FILES) config backup: ## Write all data volumes to the host. docker run --rm --volumes-from -T $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql diff --git a/README.rst b/README.rst index 426108d3c5..63a9e337f0 100644 --- a/README.rst +++ b/README.rst @@ -152,7 +152,7 @@ All of the services can be run by following the steps below. For analyticstack, make dev.sync.provision - Provision using `nfs`_: + Provision using NFS: .. code:: sh @@ -177,7 +177,7 @@ All of the services can be run by following the steps below. For analyticstack, make dev.sync.up - Start using `nfs`_: + Start using NFS: .. code:: sh @@ -539,7 +539,7 @@ master branches and ``latest`` images, unset ``OPENEDX_RELEASE`` or set it to an empty string. How do I run multiple named Open edX releases on same machine? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------------------------------- You can have multiple isolated Devstacks provisioned on a single computer now. Follow these directions to switch between the named releases. #. Bring down any running containers by issuing a `make stop.all`. @@ -548,18 +548,19 @@ You can have multiple isolated Devstacks provisioned on a single computer now. F #. Perform steps in `How do I run the images for a named Open edX release?`_ for specific release. #. Follow the steps in `Getting Started`_ section to update requirements (e.g. ``make requirements``) and provision (e.g. ``make dev.provision``) the new named release containers. -Using with OPENEDX_RELEASE: -Many of you may use the ``OPENEDX_RELEASE`` environment variable to specify the version of named release services that are checked out during Devstack provisioning. If you set ``OPENEDX_RELEASE`` and do not specify ``COMPOSE_PROJECT_NAME``, then your Devstack project name will be set as ``devstack-${OPENEDX_RELEASE}``. - As a specific example, if ``OPENEDX_RELEASE`` is set in your environment as ``juniper.master``, then ``COMPOSE_PROJECT_NAME`` will default to ``devstack-juniper.master`` instead of ``devstack``. The implication of this is that you can switch between isolated Devstack databases by changing the value of the ``OPENEDX_RELEASE`` environment variable. -Switch between your Devstack projects by doing the following: +Switch between your Devstack releases by doing the following: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +#. Bring down the containers by issuing a ``make stop.all`` for the running release. +#. Follow the instructions from the `How do I run multiple named Open edX releases on same machine?`_ section. +#. Edit the project name in ``options.local.mk`` or set the ``OPENEDX_RELEASE`` environment variable and let the ``COMPOSE_PROJECT_NAME`` be assigned automatically. +#. Bring up the containers with ``make dev.up``. -#. Bring down the containers by issuing a `make stop.all`. -#. Edit the project name in ``options.local.mk`` or set ``OPENEDX_RELEASE`` and let the ``COMPOSE_PROJECT_NAME`` be assigned automatically. -#. Bring up the containers by issuing a ``make dev.up`` or ``make dev.up.all``. +**NOTE:** Additional instructions on switching releases using `direnv` can be found in `How do I switch releases using 'direnv'?`_ section. Examples of Docker Service Names After Setting the ``COMPOSE_PROJECT_NAME`` variable. Notice that the **devstack-juniper.master** name represents the ``COMPOSE_PROJECT_NAME``. @@ -571,23 +572,48 @@ Each instance has an isolated set of databases. This could, for example, be used Unfortunately, this **does not** currently support running Devstacks simultaneously, because we hard-code host port numbers all over the place, and two running containers cannot share the same host port. Questions & Troubleshooting – Multiple Named Open edX Releases on Same Machine -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------------------------------------------ This broke my existing Devstack! -******************************** -Try posting on Discourse or Slack to see if you have the same issue as any others. If you think you have found a bug, file a CR ticket. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + See if the troubleshooting of this readme can help resolve your broken devstack first, then try posting on the `Open edX forums `__ to see if you have the same issue as any others. If you think you have found a bug, file a CR ticket. I’m getting errors related to ports already being used. -******************************************************* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Make sure you bring down your devstack before changing the value of COMPOSE_PROJECT_NAME. If you forgot to, change the COMPOSE_PROJECT_NAME back to its original value, run ``make dev.down``, and then try again. I have custom scripts/compose files that integrate with or extend Devstack. Will those still work? -************************************************************************************************** +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ With the default value of COMPOSE_PROJECT_NAME = devstack, they should still work. If you choose a different COMPOSE_PROJECT_NAME, your extensions will likely break, because the names of containers change along with the project name. - + +How do I switch releases using 'direnv'? +---------------------------------------- + +Follow directions in `Switch between your Devstack releases by doing the following:`_ then make the following adjustments. + +Make sure that you have setup each Open edX release in separate directories using `How do I enable environment variables for current directory using 'direnv'?`_ instructions. Open the next release project in a separate code editor, then activate the ``direnv`` environment variables and virtual environment for the next release by using a terminal shell to traverse to the directory with the corresponding release ``.envrc`` file. You may need to issue a ``direnv allow`` command to enable the ``.envrc`` file. + + .. code:: sh + + # You should see something like the following after successfully enabling 'direnv' for the Juniper release. + + direnv: loading ~/open-edx/devstack.juniper/.envrc + direnv: export +DEVSTACK_WORKSPACE +OPENEDX_RELEASE +VIRTUAL_ENV ~PATH + (venv)username@computer-name devstack.juniper % + +**NOTE:** Setting of the ``OPENEDX_RELEASE`` should have been handled within the ``.envrc`` file for named releases only and should not be defined for the ``master`` release. + How do I enable environment variables for current directory using 'direnv'? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Recommend separating the named releases into different directories since `direenv` unloads/loads different environment variables per directory. +--------------------------------------------------------------------------- +We recommend separating the named releases into different directories, for clarity purposes. You can use `direnv `__ to define different environment variables per directory:: + + .. code:: + + # Example showing directory structure for separate Open edX releases. + + /Users//open-edx – root directory for platform development + |_ ./devstack.master – directory containing all repository information related to the main development release. + |_ ./devstack.juniper – directory containing all repository information related to the Juniper release. #. Install `direnv` using instructions on https://direnv.net/. Below you will find additional setup at the time of this writing so refer to latest of `direnv` site for additional configuration needed. diff --git a/provision-ecommerce.sh b/provision-ecommerce.sh index 3846ad7045..bfefe42217 100755 --- a/provision-ecommerce.sh +++ b/provision-ecommerce.sh @@ -4,6 +4,6 @@ ./provision-ida.sh ecommerce ecommerce 18130 # Configure ecommerce -docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --client-id=ecommerce-key --client-secret=ecommerce-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --journals_api_url=http://journals.app:18606/api/v1/' +docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/' docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX'