From a1cecafdf8b8c660b81ffd1664536913d9e677a4 Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Tue, 26 Oct 2021 14:04:00 -0400 Subject: [PATCH 1/7] Update docker commands to be docker-compose to handle unnamed containers for multi-version devstack support https://discuss.openedx.org/t/docker-devstack-multiple-releases-one-machine/1902 Multiple devstack releases with one machine. This is in regards to this pull request #532 Updating volumes for `themes` to work with the `ironwood` release. Applying these changes to get local devstack working for ironwood release. #376 #377 Moved `COMPOSE_PROJECT_NAME` update to `options.mk` file. Additional changes to get `devstack_docker` for Ironwood working multisite environment setup. Additional changes needed to make `docker` commands work making use of the `dev.print-container.%` target to get the container id. Reverting some of these commands back to `docker` type to align with what we did with 59f812e --- Makefile | 95 +++++++++------ Makefile.edx | 10 +- course-generator/create-courses.sh | 8 +- docker-compose-analytics-pipeline.yml | 49 ++++++-- docker-compose-marketing-site.yml | 6 +- docker-compose-watchers-nfs.yml | 34 ++++++ docker-compose-watchers.yml | 12 +- docker-compose-xqueue.yml | 12 +- docker-compose.yml | 163 ++++++++++++++++++++++---- dump-db.sh | 2 +- gather-feature-toggle-state.sh | 31 +++++ load-db.sh | 3 +- marketing.mk | 2 +- options.mk | 59 ++++++++++ provision-analytics-pipeline.sh | 10 +- provision-credentials.sh | 10 +- provision-e2e.sh | 2 +- provision-ecommerce.sh | 6 +- provision-ida-user.sh | 4 +- provision-ida.sh | 8 +- provision-notes.sh | 2 +- provision-retirement-user.sh | 4 +- provision-xqueue.sh | 4 +- provision.sh | 14 +-- 24 files changed, 437 insertions(+), 113 deletions(-) create mode 100644 docker-compose-watchers-nfs.yml create mode 100644 gather-feature-toggle-state.sh create mode 100644 options.mk diff --git a/Makefile b/Makefile index 2153c0d246..4e6034019c 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,17 @@ COMPOSE_PROJECT_NAME=devstack export DEVSTACK_WORKSPACE export COMPOSE_PROJECT_NAME -include *.mk +# include *.mk + +# Include options (configurable through options.local.mk) +include options.mk + +# Include local overrides to options. +# You can use this file to configure your Devstack. It is ignored by git. +-include options.local.mk # Prefix with hyphen to tolerate absence of file. + +# Include local makefile with additional targets. +-include local.mk # Prefix with hyphen to tolerate absence of file. # Generates a help message. Borrowed from https://github.com/pydanny/cookiecutter-djangopackage. help: ## Display this help message @@ -71,6 +81,10 @@ dev.up.xqueue: | check-memory ## Bring up xqueue, assumes you already have lms r dev.up.all: | dev.up dev.up.watchers ## Bring up all services with host volumes, including watchers +dev.print-container.%: ## Get the ID of the running container for a given service. + @# Can be run as ``make --silent dev.print-container.$service`` for just ID. + @echo $$(docker-compose ps --quiet $*) + dev.sync.daemon.start: ## Start the docker-sycn daemon docker-sync start @@ -126,84 +140,93 @@ validate: ## Validate the devstack configuration docker-compose config backup: ## Write all data volumes to the host. - docker run --rm --volumes-from edx.devstack.mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql - docker run --rm --volumes-from edx.devstack.mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db - docker run --rm --volumes-from edx.devstack.elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data + docker run --rm --volumes-from -T $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql + docker run --rm --volumes-from -T $$(make -s dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db + docker run --rm --volumes-from -T $$(make -s dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data restore: ## Restore all data volumes from the host. WARNING: THIS WILL OVERWRITE ALL EXISTING DATA! - docker run --rm --volumes-from edx.devstack.mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz - docker run --rm --volumes-from edx.devstack.mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz - docker run --rm --volumes-from edx.devstack.elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz + docker run --rm --volumes-from -T $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz + docker run --rm --volumes-from -T $$(make -s dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz + docker run --rm --volumes-from -T $$(make -s dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz # TODO: Print out help for this target. Even better if we can iterate over the # services in docker-compose.yml, and print the actual service names. %-shell: ## Run a shell on the specified service container - docker exec -it edx.devstack.$* /bin/bash + docker-compose exec $* /bin/bash -credentials-shell: - docker exec -it edx.devstack.credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' +credentials-shell: ## Run a shell on the credentials container + docker-compose exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' discovery-shell: ## Run a shell on the discovery container - docker exec -it edx.devstack.discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open + docker-compose exec discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open ecommerce-shell: ## Run a shell on the ecommerce container - docker exec -it edx.devstack.ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open + docker-compose exec ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open e2e-shell: ## Start the end-to-end tests container with a shell - docker run -it --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash + docker-compose run --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash %-update-db: ## Run migrations for the specified service container - docker exec -t edx.devstack.$* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' + docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' studio-update-db: ## Run migrations for the Studio container - docker exec -t edx.devstack.studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' + docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' lms-update-db: ## Run migrations LMS container - docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' + docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' update-db: | studio-update-db lms-update-db discovery-update-db ecommerce-update-db credentials-update-db ## Run the migrations for all services lms-shell: ## Run a shell on the LMS container - docker exec -it edx.devstack.lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose exec lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open lms-watcher-shell: ## Run a shell on the LMS watcher container - docker exec -it edx.devstack.lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose exec lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open %-attach: ## Attach to the specified service container process to use the debugger & see logs. - docker attach edx.devstack.$* + docker attach "$$(make --silent dev.print-container.$*)" lms-restart: ## Kill the LMS Django development server. The watcher process will restart it. - docker exec -t edx.devstack.lms bash -c 'kill $$(ps aux | grep "manage.py lms" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose exec lms bash -c 'kill $$(ps aux | grep "manage.py lms" | egrep -v "while|grep" | awk "{print \$$2}")' studio-shell: ## Run a shell on the Studio container - docker exec -it edx.devstack.studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose exec studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-watcher-shell: ## Run a shell on the studio watcher container - docker exec -it edx.devstack.studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose exec studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-restart: ## Kill the LMS Django development server. The watcher process will restart it. - docker exec -t edx.devstack.studio bash -c 'kill $$(ps aux | grep "manage.py cms" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose exec studio bash -c 'kill $$(ps aux | grep "manage.py cms" | egrep -v "while|grep" | awk "{print \$$2}")' xqueue-shell: ## Run a shell on the XQueue container - docker exec -it edx.devstack.xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open + docker-compose exec xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue-restart: ## Kill the XQueue development server. The watcher process will restart it. - docker exec -t edx.devstack.xqueue bash -c 'kill $$(ps aux | grep "manage.py runserver" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose exec xqueue bash -c 'kill $$(ps aux | grep "manage.py runserver" | egrep -v "while|grep" | awk "{print \$$2}")' + +%-restart-devserver: ## Kill a service's Django development server. The watcher process should restart it. + docker-compose exec $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' xqueue_consumer-shell: ## Run a shell on the XQueue consumer container - docker exec -it edx.devstack.xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open + docker-compose exec xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue_consumer-restart: ## Kill the XQueue development server. The watcher process will restart it. - docker exec -t edx.devstack.xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose exec xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")' %-static: ## Rebuild static assets for the specified service container - docker exec -t edx.devstack.$* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' + docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' + +lms-sass: ## Rebuild sass assets for the LMS container + docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=lms --theme-dirs /edx/app/edx-themes/edx-platform/ --themes my-theme' lms-static: ## Rebuild static assets for the LMS container - docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/ --themes educateworkforce greermade' + docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/ --themes educateworkforce greermade' + +studio-sass: ## Rebuild sass assets for the Studio container + docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=cms --theme-dirs /edx/app/edx-themes/edx-platform/ --themes my-theme' studio-static: ## Rebuild static assets for the Studio container (educateworkforce,arm,belizeproject,bmw,caregiver,dlsc,greermade,mahcc,meep,nccte,nwgrc,ptc,scelect,scjru,ts) - docker exec -t edx.devstack.studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/ --themes educateworkforce greermade' + docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/ --themes educateworkforce greermade' static: | credentials-static discovery-static ecommerce-static lms-static studio-static ## Rebuild static assets for all service containers @@ -215,12 +238,12 @@ e2e-tests: ## Run the end-to-end tests against the service containers validate-lms-volume: ## Validate that changes to the local workspace are reflected in the LMS container touch $(DEVSTACK_WORKSPACE)/edx-platform/testfile - docker exec edx.devstack.lms ls /edx/app/edxapp/edx-platform/testfile + docker-compose exec -T lms ls /edx/app/edxapp/edx-platform/testfile rm $(DEVSTACK_WORKSPACE)/edx-platform/testfile vnc-passwords: ## Get the VNC passwords for the Chrome and Firefox Selenium containers - @docker logs edx.devstack.chrome 2>&1 | grep "VNC password" | tail -1 - @docker logs edx.devstack.firefox 2>&1 | grep "VNC password" | tail -1 + @docker-compose logs chrome 2>&1 | grep "VNC password" | tail -1 + @docker-compose logs firefox 2>&1 | grep "VNC password" | tail -1 devpi-password: ## Get the root devpi password for the devpi container docker-compose exec devpi bash -c "cat /data/server/.serverpassword" @@ -242,7 +265,7 @@ dev.provision.analytics_pipeline.run: DOCKER_COMPOSE_FILES="-f docker-compose.yml -f docker-compose-host.yml -f docker-compose-analytics-pipeline.yml" ./provision-analytics-pipeline.sh analytics-pipeline-shell: ## Run a shell on the analytics pipeline container - docker exec -it edx.devstack.analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open + docker-compose exec analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open dev.up.analytics_pipeline: | check-memory ## Bring up analytics pipeline services docker-compose -f docker-compose.yml -f docker-compose-analytics-pipeline.yml -f docker-compose-host.yml up -d analyticspipeline @@ -251,14 +274,14 @@ pull.analytics_pipeline: ## Update analytics pipeline docker images docker-compose -f docker-compose.yml -f docker-compose-analytics-pipeline.yml pull --parallel analytics-pipeline-devstack-test: ## Run analytics pipeline tests in travis build - docker exec -u hadoop -i edx.devstack.analytics_pipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity' + docker-compose $(DOCKER_COMPOSE_FILES) exec -u hadoop -T analytics_pipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity' stop.analytics_pipeline: ## Stop analytics pipeline services docker-compose -f docker-compose.yml -f docker-compose-analytics-pipeline.yml stop docker-compose up -d mysql ## restart mysql as other containers need it hadoop-application-logs-%: ## View hadoop logs by application Id - docker exec -it edx.devstack.analytics_pipeline.nodemanager yarn logs -applicationId $* + docker-compose exec analytics_pipeline.nodemanager yarn logs -applicationId $* # Provisions studio, ecommerce, and marketing with course(s) in test-course.json # Modify test-course.json before running this make target to generate a custom course diff --git a/Makefile.edx b/Makefile.edx index fd9f8ac83d..9ce4a6c1d3 100644 --- a/Makefile.edx +++ b/Makefile.edx @@ -23,12 +23,12 @@ dev.provision.whitelabel: # The containers must be started with the 'dev.up.e2e_wl_tests' target. # AND the test must be setup using the 'dev.provision.whitelabel' target. whitelabel-tests: - docker run -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e - docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh devstack.whitelabel:/tmp/run_whitelabel_tests.sh - docker exec -t devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh + docker-compose run -T -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e + docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh $(make --silent dev.print-container.devstack.whitelabel)":/tmp/run_whitelabel_tests.sh + docker-compose exec devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh whitelabel-cleanup: - docker rm --force devstack.whitelabel + docker-compose rm --force devstack.whitelabel whitelabel-shell: ## Start a whitelabel test shell session - docker exec -it devstack.whitelabel env TERM=$(TERM) bash + docker-compose exec -it devstack.whitelabel env TERM=$(TERM) bash diff --git a/course-generator/create-courses.sh b/course-generator/create-courses.sh index eea96f6589..b7ee059a9e 100755 --- a/course-generator/create-courses.sh +++ b/course-generator/create-courses.sh @@ -15,14 +15,14 @@ for arg in "$@"; do studio=true fi elif [ $arg == "--ecommerce" ]; then - if [ ! "$(docker exec -t edx.devstack.ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then + if [ ! "$(docker-compose exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then echo "Issue with ecommerce container" container_error=true else ecommerce=true fi elif [ $arg == "--marketing" ]; then - if [ ! "$(docker exec -t edx.devstack.marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then + if [ ! "$(docker-compose exec marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then echo "Issue with marketing container. Course creation will proceed without marketing container." else marketing=true @@ -54,10 +54,10 @@ fi if $ecommerce ; then echo "Creating courses on ecommerce." - docker exec -t edx.devstack.ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" + docker-compose exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" fi if $marketing ; then echo "Creating courses on marketing." - docker exec -t edx.devstack.marketing bash -c "drush generate_courses '$course_json'" + docker-compose exec marketing bash -c "drush generate_courses '$course_json'" fi diff --git a/docker-compose-analytics-pipeline.yml b/docker-compose-analytics-pipeline.yml index eef6b6fa24..8f0a546bc3 100644 --- a/docker-compose-analytics-pipeline.yml +++ b/docker-compose-analytics-pipeline.yml @@ -2,11 +2,15 @@ version: "2.1" services: namenode: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.namenode" image: edxops/analytics_pipeline_hadoop_namenode:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.namenode hostname: namenode environment: - CLUSTER_NAME=devstack + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.namenode ports: - 127.0.0.1:50070:50070 command: ["/run.sh"] @@ -14,13 +18,17 @@ services: - namenode_data:/hadoop/dfs/name datanode: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.datanode" image: edxops/analytics_pipeline_hadoop_datanode:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.datanode hostname: datanode environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" depends_on: - namenode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.datanode ports: - 127.0.0.1:50075:50075 command: ["/run.sh"] @@ -28,8 +36,8 @@ services: - datanode_data:/hadoop/dfs/data resourcemanager: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.resourcemanager" image: edxops/analytics_pipeline_hadoop_resourcemanager:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.resourcemanager hostname: resourcemanager environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" @@ -40,13 +48,17 @@ services: depends_on: - namenode - datanode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.resourcemanager ports: - 127.0.0.1:8088:8088 # resource manager web ui command: ["/run.sh"] nodemanager: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.nodemanager" image: edxops/analytics_pipeline_hadoop_nodemanager:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.nodemanager hostname: nodemanager environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" @@ -60,15 +72,23 @@ services: - resourcemanager - namenode - datanode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.nodemanager ports: - 127.0.0.1:8042:8042 # node manager web ui - 127.0.0.1:19888:19888 # node manager job history server ui command: ["/run.sh"] sparkmaster: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkmaster" image: edxops/analytics_pipeline_spark_master:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.sparkmaster hostname: sparkmaster + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.sparkmaster ports: - 127.0.0.1:8080:8080 - 127.0.0.1:7077:7077 # spark master port @@ -76,25 +96,34 @@ services: - 127.0.0.1:18080:18080 # spark history server sparkworker: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkworker" image: edxops/analytics_pipeline_spark_worker:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.sparkworker hostname: sparkworker depends_on: - sparkmaster environment: - SPARK_MASTER=spark://sparkmaster:7077 + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.sparkworker ports: - 127.0.0.1:8081:8081 # spark worker UI vertica: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.vertica" image: sumitchawla/vertica:latest - container_name: edx.devstack.analytics_pipeline.vertica + hostname: vertica + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.vertica volumes: - vertica_data:/home/dbadmin/docker analyticspipeline: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline" image: edxops/analytics_pipeline:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline hostname: analyticspipeline volumes: - ${DEVSTACK_WORKSPACE}/edx-analytics-pipeline:/edx/app/analytics_pipeline/analytics_pipeline @@ -108,6 +137,10 @@ services: - sparkworker - elasticsearch - vertica + networks: + default: + aliases: + - edx.devstack.analytics_pipeline ports: - 127.0.0.1:4040:4040 # spark web UI environment: diff --git a/docker-compose-marketing-site.yml b/docker-compose-marketing-site.yml index 4bd2693b7a..bc48b89ce1 100644 --- a/docker-compose-marketing-site.yml +++ b/docker-compose-marketing-site.yml @@ -7,7 +7,7 @@ services: - MARKETING_SITE_ROOT="http://localhost:8080" marketing: - container_name: edx.devstack.marketing + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.marketing" depends_on: - mysql - memcached @@ -26,5 +26,9 @@ services: # IP address of your machine to enable debugging (IP_ADDRESS set in .env file) - XDEBUG_CONFIG=remote_host=${XDEBUG_IP_ADDRESS:-127.0.0.1} image: edxops/edx-mktg:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.marketing ports: - "8080:80" diff --git a/docker-compose-watchers-nfs.yml b/docker-compose-watchers-nfs.yml new file mode 100644 index 0000000000..532a49060f --- /dev/null +++ b/docker-compose-watchers-nfs.yml @@ -0,0 +1,34 @@ +version: "2.1" + +services: + lms_watcher: + command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' + container_name: edx.devstack.lms_watcher + environment: + BOK_CHOY_HOSTNAME: edx.devstack.lms_watcher + ASSET_WATCHER_TIMEOUT: 12 + image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + volumes: + - edx-nfs:/edx/app/edxapp/edx-platform + - edxapp_lms_assets:/edx/var/edxapp/staticfiles/ + - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules + - src-nfs:/edx/src + - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes:cached + + studio_watcher: + command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' + container_name: edx.devstack.studio_watcher + environment: + BOK_CHOY_HOSTNAME: edx.devstack.studio_watcher + ASSET_WATCHER_TIMEOUT: 12 + image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + volumes: + - edxapp_studio_assets:/edx/var/edxapp/staticfiles/ + - edx-nfs:/edx/app/edxapp/edx-platform + - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules + - src-nfs:/edx/src:cached + - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes:cached +volumes: + edxapp_lms_assets: + edxapp_studio_assets: + edxapp_node_modules: diff --git a/docker-compose-watchers.yml b/docker-compose-watchers.yml index 0f55930d19..f464815f43 100644 --- a/docker-compose-watchers.yml +++ b/docker-compose-watchers.yml @@ -3,11 +3,15 @@ version: "2.1" services: lms_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT --theme-dirs=/edx/app/edx-themes/edx-platform/ --themes=educateworkforce,greermade; sleep 2; done' - container_name: edx.devstack.lms_watcher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.lms_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.lms_watcher volumes: - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform:cached - edxapp_lms_assets:/edx/var/edxapp/staticfiles/ @@ -17,11 +21,15 @@ services: studio_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT --theme-dirs=/edx/app/edx-themes/edx-platform/ --themes=educateworkforce,greermade; sleep 2; done' - container_name: edx.devstack.studio_watcher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.studio_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.studio_watcher volumes: - edxapp_studio_assets:/edx/var/edxapp/staticfiles/ - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform:cached diff --git a/docker-compose-xqueue.yml b/docker-compose-xqueue.yml index bf4a73ac54..89e987c5ba 100644 --- a/docker-compose-xqueue.yml +++ b/docker-compose-xqueue.yml @@ -2,19 +2,27 @@ version: "2.1" services: xqueue: - container_name: edx.devstack.xqueue + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue" image: edxops/xqueue:${OPENEDX_RELEASE:-latest} command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py runserver 0.0.0.0:18040 ; sleep 2; done' volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file + networks: + default: + aliases: + - edx.devstack.xqueue ports: - 18040:18040 xqueue_consumer: - container_name: edx.devstack.xqueue_consumer + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue_consumer" image: edxops/xqueue:${OPENEDX_RELEASE:-latest} command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py run_consumer ; sleep 2; done' + networks: + default: + aliases: + - edx.devstack.xqueue_consumer volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file diff --git a/docker-compose.yml b/docker-compose.yml index 027bac2e94..edf0277105 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,9 +13,14 @@ version: "2.1" services: # Third-party services chrome: - container_name: edx.devstack.chrome + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.chrome" + hostname: chrome.devstack.edx image: edxops/chrome:${OPENEDX_RELEASE:-latest} shm_size: 2g + networks: + default: + aliases: + - edx.devstack.chrome ports: - "15900:5900" volumes: # for file uploads @@ -23,8 +28,13 @@ services: - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data elasticsearch: - container_name: edx.devstack.elasticsearch + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.elasticsearch" + hostname: elasticsearch.devstack.edx image: edxops/elasticsearch:devstack + networks: + default: + aliases: + - edx.devstack.elasticsearch # TODO: What to do about these forwarded ports? They'll conflict with ports forwarded by the Vagrant VM. # ports: # - "9200:9200" @@ -33,10 +43,21 @@ services: - elasticsearch_data:/usr/share/elasticsearch/data - elasticsearch_data:/usr/share/elasticsearch/logs + # This is meant to be used to test ES upgrades so that we do not have to upgrade all of our services to ES5 at once. + elasticsearch-5: + container_name: edx.devstack.elasticsearch-5 + hostname: elasticsearch-5.devstack.edx + image: elasticsearch:5.6.16 + firefox: - container_name: edx.devstack.firefox + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.firefox" + hostname: firefox.devstack.edx image: edxops/firefox:${OPENEDX_RELEASE:-latest} shm_size: 2g + networks: + default: + aliases: + - edx.devstack.firefox ports: - "25900:5900" volumes: # for file uploads @@ -44,8 +65,13 @@ services: - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data memcached: - container_name: edx.devstack.memcached + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.memcached" + hostname: memcached.devstack.edx image: memcached:1.4.24 + networks: + default: + aliases: + - edx.devstack.memcached # ports: # - "11211:11211" @@ -54,8 +80,13 @@ services: # to conserve disk space, and disable the journal for a minor performance gain. # See https://docs.mongodb.com/v3.0/reference/program/mongod/#options for complete details. command: mongod --smallfiles --nojournal --storageEngine wiredTiger - container_name: edx.devstack.mongo + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mongo" + hostname: mongo.devstack.edx image: mongo:3.2.16 + networks: + default: + aliases: + - edx.devstack.mongo # ports: # - "27017:27017" volumes: @@ -63,20 +94,33 @@ services: mysql: command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci - container_name: edx.devstack.mysql + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql" + hostname: mysql.devstack.edx environment: MYSQL_ROOT_PASSWORD: "" MYSQL_ALLOW_EMPTY_PASSWORD: "yes" image: mysql:5.6 + networks: + default: + aliases: + - edx.devstack.mysql ports: - "33060:3306" volumes: - mysql_data:/var/lib/mysql + redis: + container_name: edx.devstack.redis + hostname: redis.devstack.edx + image: redis:2.8 + command: redis-server --requirepass password + + # ================================================ # edX services credentials: command: bash -c 'source /edx/app/credentials/credentials_env && while true; do python /edx/app/credentials/credentials/manage.py runserver 0.0.0.0:18150; sleep 2; done' - container_name: edx.devstack.credentials + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.credentials" + hostname: credentials.devstack.edx depends_on: - mysql - memcached @@ -89,12 +133,17 @@ services: SOCIAL_AUTH_EDX_OIDC_URL_ROOT: http://edx.devstack.lms:18000/oauth2 ENABLE_DJANGO_TOOLBAR: 1 image: edxops/credentials:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.credentials ports: - "18150:18150" discovery: command: bash -c 'source /edx/app/discovery/discovery_env && while true; do python /edx/app/discovery/discovery/manage.py runserver 0.0.0.0:18381; sleep 2; done' - container_name: edx.devstack.discovery + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.discovery" + hostname: discovery.devstack.edx depends_on: - mysql - elasticsearch @@ -106,6 +155,10 @@ services: TEST_ELASTICSEARCH_URL: "http://edx.devstack.elasticsearch:9200" ENABLE_DJANGO_TOOLBAR: 1 image: edxops/discovery:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.discovery ports: - "18381:18381" volumes: @@ -113,7 +166,8 @@ services: ecommerce: command: bash -c 'source /edx/app/ecommerce/ecommerce_env && while true; do python /edx/app/ecommerce/ecommerce/manage.py runserver 0.0.0.0:18130; sleep 2; done' - container_name: edx.devstack.ecommerce + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.ecommerce" + hostname: ecommerce.devstack.edx depends_on: - mysql - memcached @@ -123,12 +177,33 @@ services: environment: ENABLE_DJANGO_TOOLBAR: 0 image: edxops/ecommerce:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.ecommerce ports: - "18130:18130" + forum: + command: bash -c 'source /edx/app/forum/ruby_env && source /edx/app/forum/devstack_forum_env && cd /edx/app/forum/cs_comments_service && bundle install && while true; do ruby app.rb -o 0.0.0.0 ; sleep 2; done' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.forum" + hostname: forum.devstack.edx + depends_on: + - mongo + - memcached + - elasticsearch + image: edxops/forum:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.forum + ports: + - "44567:4567" + lms: command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack_docker; sleep 2; done' - container_name: edx.devstack.lms + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms" + hostname: lms.devstack.edx depends_on: - devpi - mysql @@ -144,6 +219,10 @@ services: EDXAPP_TEST_MONGO_HOST: edx.devstack.mongo NO_PYTHON_UNINSTALL: 1 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.lms ports: - "18000:18000" - "19876:19876" # JS test debugging @@ -154,12 +233,17 @@ services: edx_notes_api: command: bash -c 'source /edx/app/edx_notes_api/edx_notes_api_env && while true; do python /edx/app/edx_notes_api/edx_notes_api/manage.py runserver 0.0.0.0:18120 --settings notesserver.settings.devstack; sleep 2; done' - container_name: edx.devstack.edx_notes_api + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.edx_notes_api" + hostname: edx_notes_api.devstack.edx depends_on: - devpi - elasticsearch - mysql image: edxops/notes:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.edxnotesapi ports: - "18120:18120" environment: @@ -169,12 +253,14 @@ services: DB_PASSWORD: "password" DB_PORT: "3306" DB_USER: "notes001" + DJANGO_WATCHMAN_TIMEOUT: 30 ENABLE_DJANGO_TOOLBAR: 1 ELASTICSEARCH_URL: "http://edx.devstack.elasticsearch:9200" studio: command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py cms runserver 0.0.0.0:18010 --settings devstack_docker; sleep 2; done' - container_name: edx.devstack.studio + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio" + hostname: studio.devstack.edx depends_on: - devpi - mysql @@ -190,6 +276,10 @@ services: EDXAPP_TEST_MONGO_HOST: edx.devstack.mongo NO_PYTHON_UNINSTALL: 1 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.studio ports: - "18010:18010" - "19877:19877" # JS test debugging @@ -198,25 +288,58 @@ services: volumes: - edxapp_studio_assets:/edx/var/edxapp/staticfiles/ - forum: - command: bash -c 'source /edx/app/forum/ruby_env && source /edx/app/forum/devstack_forum_env && cd /edx/app/forum/cs_comments_service && bundle install && while true; do ruby app.rb -o 0.0.0.0 ; sleep 2; done' - container_name: edx.devstack.forum + # ========================================================================== + # edX Microfrontends + # + # TODO: Instead of having an nginx container for every single microfrontend + # (there are like 12 now, right?), we should come up with an actual strategy + # for micro-frontends in devtack. + # ========================================================================== + + frontend-app-publisher: + extends: + file: microfrontend.yml + service: microfrontend + working_dir: '/edx/app/frontend-app-publisher' + container_name: edx.devstack.frontend-app-publisher + ports: + - "18400:18400" depends_on: - - mongo - - memcached - - elasticsearch - image: edxops/forum:${OPENEDX_RELEASE:-latest} + - lms + + gradebook: + extends: + file: microfrontend.yml + service: microfrontend + working_dir: '/edx/app/gradebook' + container_name: edx.devstack.gradebook ports: - "44567:4567" devpi: - container_name: edx.devstack.devpi + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.devpi" + hostname: devpi.devstack.edx image: edxops/devpi:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.devpi ports: - "3141:3141" volumes: - devpi_data:/data + frontend-app-learning: + extends: + file: microfrontend.yml + service: microfrontend + working_dir: '/edx/app/frontend-app-learning' + container_name: edx.devstack.frontend-app-learning + ports: + - "2000:2000" + depends_on: + - lms + volumes: discovery_assets: edxapp_lms_assets: diff --git a/dump-db.sh b/dump-db.sh index 9d9c135704..50e22948fb 100755 --- a/dump-db.sh +++ b/dump-db.sh @@ -14,5 +14,5 @@ then fi echo "Dumping the $1 database..." -docker exec -i edx.devstack.mysql mysqldump --add-drop-database --skip-add-drop-table -B $1 > $1.sql +docker-compose exec mysql base -c "mysqldump --add-drop-database --skip-add-drop-table -B $1" > $1.sql echo "Finished dumping the $1 database!" diff --git a/gather-feature-toggle-state.sh b/gather-feature-toggle-state.sh new file mode 100644 index 0000000000..2ed9946195 --- /dev/null +++ b/gather-feature-toggle-state.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# gather-feature-toggle-state.sh + +# as part of our feature toggle reporting utility, this script can be run +# against a running devstack to collect the state of all of feature toggles +# (django-waffle, waffle-utils) currently configured for each of the IDAs +# that use them. + +if [ -e feature-toggle-data ]; then + rm -Rf feature-toggle-data +fi +mkdir feature-toggle-data + +docker exec -t edx.devstack.credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py dumpdata waffle --format=json > /edx/app/credentials/credentials/credentials_waffle.json' +docker cp edx.devstack.credentials:/edx/app/credentials/credentials/credentials_waffle.json feature-toggle-data +docker exec -t edx.devstack.credentials bash -c 'rm /edx/app/credentials/credentials/credentials_waffle.json' + +docker exec -t edx.devstack.discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py dumpdata waffle --format=json > /edx/app/discovery/discovery/discovery_waffle.json' +docker cp edx.devstack.discovery:/edx/app/discovery/discovery/discovery_waffle.json feature-toggle-data +docker exec -t edx.devstack.discovery bash -c '> /edx/app/discovery/discovery/discovery_waffle.json' + +docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py dumpdata waffle --format=json > /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' +docker cp edx.devstack.ecommerce:/edx/app/ecommerce/ecommerce/ecommerce_waffle.json feature-toggle-data +docker exec -t edx.devstack.ecommerce bash -c 'rm /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' + +docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle --format=json > /edx/app/edxapp/edx-platform/lms_waffle.json' +docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle_utils --format=json > /edx/app/edxapp/edx-platform/lms_waffle_utils.json' +docker cp edx.devstack.lms:/edx/app/edxapp/edx-platform/lms_waffle.json feature-toggle-data +docker cp edx.devstack.lms:/edx/app/edxapp/edx-platform/lms_waffle_utils.json feature-toggle-data +docker exec -t edx.devstack.lms bash -c 'rm /edx/app/edxapp/edx-platform/lms_waffle*.json' diff --git a/load-db.sh b/load-db.sh index 24b70fada5..9b9aaa6c75 100755 --- a/load-db.sh +++ b/load-db.sh @@ -17,5 +17,6 @@ then fi echo "Loading the $1 database..." -docker exec -i edx.devstack.mysql mysql -uroot $1 < $1.sql +mysql_container=$(make -s dev.print-container.mysql) +docker exec "$mysql_container" bash -c "mysql -uroot $1" < $1.sql echo "Finished loading the $1 database!" diff --git a/marketing.mk b/marketing.mk index 6932e36395..765a47adc5 100644 --- a/marketing.mk +++ b/marketing.mk @@ -3,7 +3,7 @@ help-marketing: ## Display this help message @perl -nle'print $& if m{^[\.a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | grep marketing | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m %-25s\033[0m %s\n", $$1, $$2}' marketing-shell: ## Run a shell on the marketing site container - docker exec -it edx.devstack.marketing env TERM=$(TERM) bash + docker-compose exec marketing env TERM=$(TERM) bash -c 'cd /edx/app/edx-mktg/edx-mktg; exec /bin/bash -sh' stop-marketing: ## Stop all services (including the marketing site) with host volumes docker-compose -f docker-compose.yml -f docker-compose-host.yml -f docker-compose-marketing-site.yml -f docker-compose-marketing-site-host.yml stop diff --git a/options.mk b/options.mk new file mode 100644 index 0000000000..0e0a456099 --- /dev/null +++ b/options.mk @@ -0,0 +1,59 @@ +# DEFAULT DEVSTACK OPTIONS +# Included into Makefile and exported to command environment. +# Defaults are listed in this file. +# Local git-ignored overrides can be configured by creating `options.local.mk`. +# Variables are set here with ?= to allow for overriding them on the command line. + +# Folder in which we looks for repositories. +# Defaults to parent. +DEVSTACK_WORKSPACE ?= $(shell pwd)/.. + +# Name of Docker Compose project. +# See https://docs.docker.com/compose/reference/envvars/#compose_project_name +# Defaults to 'devstack' should OPENEDX_RELEASE not be defined. +ifdef OPENEDX_RELEASE + COMPOSE_PROJECT_NAME ?= devstack-${OPENEDX_RELEASE} +else + COMPOSE_PROJECT_NAME ?= devstack +endif + +# Name of Docker Compose project. +# See https://docs.docker.com/compose/reference/envvars/#compose_project_name +# Defaults to 'devstack'. +COMPOSE_PROJECT_NAME ?= devstack + +# increase Docker Compose HTTP timeout so that devstack provisioning does not fail in unstable networks +COMPOSE_HTTP_TIMEOUT=180 + +# Whether we should always copy programs to LMS cache upon LMS startup. +# If 'true', then run `make dev.cache-programs` whenever we bring up +# containers. +# Defaults to false. +ALWAYS_CACHE_PROGRAMS ?= false + +# FileSystem Synchronization Strategy. +# How should we synchronize files between the host machine and the Docker containers? +# Options are 'local-mount', 'nfs', and 'docker-sync'. +# Note that 'local-mount' is the most tested and supported with edX's Devstack +# and 'docker-sync' the least. +FS_SYNC_STRATEGY ?= local-mounts + +# Services that are pulled, provisioned, run, and checked by default +# when no services are specified manually. +# TODO: Re-evaluate this list and consider paring it down to a tighter core. +# The current value was chosen such that it would not change the existing +# Devstack behavior. +DEFAULT_SERVICES ?= lms+studio+ecommerce+discovery+credentials+forum+edx_notes_api+gradebook+frontend-app-publisher + +# List of all services with database migrations. +# Services must provide a Makefile target named: $(service)-update-db +# Note: This list should contain _all_ db-backed services, even if not +# configured to run; the list will be filtered later against $(DEFAULT_SERVICES) +DB_SERVICES ?= credentials discovery ecommerce lms registrar studio + +# List of Makefile targets to run database migrations, in the form $(service)-update-db +# Services will only have their migrations added here +# if the service is present in both $(DEFAULT_SERVICES) and $(DB_SERVICES). +DB_MIGRATION_TARGETS = $(foreach db_service,$(DB_SERVICES),\ + $(if $(filter $(db_service),$(subst +, ,$(DEFAULT_SERVICES))),\ + $(db_service)-update-db)) diff --git a/provision-analytics-pipeline.sh b/provision-analytics-pipeline.sh index a866fe3fe6..3f00f1892c 100755 --- a/provision-analytics-pipeline.sh +++ b/provision-analytics-pipeline.sh @@ -21,7 +21,7 @@ docker-compose $DOCKER_COMPOSE_FILES up -d mysql analyticspipeline # Ensure the MySQL server is online and usable echo "Waiting for MySQL" -until docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')" &> /dev/null +until docker-compose exec mysql mysql -uroot -se "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')" &> /dev/null do printf "." sleep 1 @@ -34,12 +34,12 @@ sleep 20 # Analytics pipeline has dependency on lms but we only need its db schema & not full lms. So we'll just load their db # schemas as part of analytics pipeline provisioning. If there is a need of a fully fledged LMS, then provision lms # by following their documentation. -if [[ ! -z "`docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='edxapp'" 2>&1`" ]]; +if [[ ! -z "`docker-compose exec mysql mysql -uroot -se "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='edxapp'" 2>&1`" ]]; then echo -e "${GREEN}LMS DB exists, skipping lms schema load.${NC}" else echo -e "${GREEN}LMS DB not found, provisioning lms schema.${NC}" - docker exec -i edx.devstack.mysql mysql -uroot mysql < provision.sql + docker-compose exec mysql bash -c 'mysql -uroot mysql' < provision.sql ./load-db.sh edxapp docker-compose $DOCKER_COMPOSE_FILES up -d lms studio docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' @@ -51,7 +51,7 @@ fi echo -e "${GREEN}LMS database provisioned successfully...${NC}" echo -e "${GREEN}Creating databases and users...${NC}" -docker exec -i edx.devstack.mysql mysql -uroot mysql < provision-analytics-pipeline.sql +docker-compose exec mysql bash -c 'mysql -uroot mysql' < provision-analytics-pipeline.sql # initialize hive metastore echo -e "${GREEN}Initializing HIVE metastore...${NC}" @@ -68,7 +68,7 @@ done sleep 10 # for datanode & other services to activate echo -e "${GREEN}Namenode is ready!${NC}" -docker exec -u hadoop -i edx.devstack.analytics_pipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' +docker-compose exec -u hadoop analytics_pipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' docker image prune -f diff --git a/provision-credentials.sh b/provision-credentials.sh index 32fccdaf61..0334f2966b 100755 --- a/provision-credentials.sh +++ b/provision-credentials.sh @@ -9,20 +9,20 @@ port=18150 docker-compose $DOCKER_COMPOSE_FILES up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" +docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" echo -e "${GREEN}Configuring site for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' +docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker-compose exec ${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-e2e.sh b/provision-e2e.sh index 47417df310..caadd418f5 100755 --- a/provision-e2e.sh +++ b/provision-e2e.sh @@ -10,7 +10,7 @@ elif [ ! -d "$DEVSTACK_WORKSPACE" ]; then fi # Copy the test course tarball into the studio container -docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz edx.devstack.studio:/tmp/ +docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz "$(make --silent dev.print-container.studio)":/tmp/ # Extract the test course tarball docker-compose exec studio bash -c 'cd /tmp && tar xzf course.tar.gz' diff --git a/provision-ecommerce.sh b/provision-ecommerce.sh index b807af9c29..00068b6c0e 100755 --- a/provision-ecommerce.sh +++ b/provision-ecommerce.sh @@ -4,6 +4,6 @@ ./provision-ida.sh ecommerce ecommerce 18130 # Configure ecommerce -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --client-id=ecommerce-key --client-secret=ecommerce-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/' -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' +docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --client-id=ecommerce-key --client-secret=ecommerce-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --journals_api_url=http://journals.app:18606/api/v1/' +docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' +docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' diff --git a/provision-ida-user.sh b/provision-ida-user.sh index a18ce1d301..57460b590c 100755 --- a/provision-ida-user.sh +++ b/provision-ida-user.sh @@ -5,5 +5,5 @@ client_name=$2 client_port=$3 echo -e "${GREEN}Creating service user and OAuth client for ${app_name}...${NC}" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_oauth2_client "http://localhost:$3" "http://localhost:$3/complete/edx-oidc/" confidential --client_name $2 --client_id "$1-key" --client_secret "$1-secret" --trusted --logout_uri "http://localhost:$3/logout/" --username $1_worker' -- "$app_name" "$client_name" "$client_port" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_oauth2_client "http://localhost:$3" "http://localhost:$3/complete/edx-oidc/" confidential --client_name $2 --client_id "$1-key" --client_secret "$1-secret" --trusted --logout_uri "http://localhost:$3/logout/" --username $1_worker' -- "$app_name" "$client_name" "$client_port" diff --git a/provision-ida.sh b/provision-ida.sh index 02206bd735..a0b38ab5ce 100755 --- a/provision-ida.sh +++ b/provision-ida.sh @@ -5,17 +5,17 @@ client_port=$3 # The port corresponding to this IDA service in devstack. docker-compose $DOCKER_COMPOSE_FILES up -d $app_name echo -e "${GREEN}Installing requirements for ${app_name}...${NC}" -docker exec -t edx.devstack.${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" +docker-compose exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" echo -e "${GREEN}Running migrations for ${app_name}...${NC}" -docker exec -t edx.devstack.${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" +docker-compose exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" echo -e "${GREEN}Creating super-user for ${app_name}...${NC}" -docker exec -t edx.devstack.${app_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" +docker-compose exec ${app_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" ./provision-ida-user.sh $app_name $client_name $client_port # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${app_name}...${NC}" -docker exec -t edx.devstack.${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" +docker-compose exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" diff --git a/provision-notes.sh b/provision-notes.sh index 4a2c924ab4..a84c077a22 100755 --- a/provision-notes.sh +++ b/provision-notes.sh @@ -5,4 +5,4 @@ # This will build the elasticsearch index for notes. echo -e "${GREEN}Creating indexes for edx_notes_api...${NC}" -docker exec -t edx.devstack.edx_notes_api bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py rebuild_index --noinput' -- edx_notes_api +docker-compose exec edx_notes_api bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py rebuild_index --noinput' -- edx_notes_api diff --git a/provision-retirement-user.sh b/provision-retirement-user.sh index b3a5dac2e3..3267258628 100755 --- a/provision-retirement-user.sh +++ b/provision-retirement-user.sh @@ -4,5 +4,5 @@ app_name=$1 user_name=$2 echo -e "${GREEN}Creating retirement service user ${user_name} and DOT Application ${app_name}...${NC}" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" +docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" diff --git a/provision-xqueue.sh b/provision-xqueue.sh index 36e55579bb..b1cdc93f59 100755 --- a/provision-xqueue.sh +++ b/provision-xqueue.sh @@ -11,13 +11,13 @@ docker-compose up -d mysql # Ensure the MySQL server is online and usable echo "Waiting for MySQL" -until docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')" &> /dev/null +until docker-compose $DOCKER_COMPOSE_FILES exec -T mysql bash -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 done -docker exec -i edx.devstack.mysql mysql -uroot mysql < provision-xqueue.sql +docker-compose $DOCKER_COMPOSE_FILES exec -T mysql bash -c "mysql -uroot mysql" < provision-xqueue.sql # Run migrations docker-compose $DOCKER_COMPOSE_FILES exec xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py migrate' # Add users that graders use to fetch data, there's one default user in Ansible which is part of our settings diff --git a/provision.sh b/provision.sh index 8324ff18dc..adaff4246e 100755 --- a/provision.sh +++ b/provision.sh @@ -21,8 +21,8 @@ NC='\033[0m' # No Color docker-compose up -d mysql mongo # Ensure the MySQL server is online and usable -echo "Waiting for MySQL" -until docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')" &> /dev/null +echo "${GREEN}Waiting for MySQL.${NC}" +until docker-compose exec -T mysql bash -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 @@ -32,11 +32,11 @@ done # This can be removed once https://github.com/docker-library/mysql/issues/245 is resolved. sleep 20 -echo -e "MySQL ready" - -echo -e "${GREEN}Creating databases and users...${NC}" -docker exec -i edx.devstack.mysql mysql -uroot mysql < provision.sql -docker exec -i edx.devstack.mongo mongo < mongo-provision.js +# Ensure that the MySQL databases and users are created for all IDAs. +# (A no-op for databases and users that already exist). +echo -e "${GREEN}Ensuring MySQL databases and users exist...${NC}" +docker-compose exec -T mysql bash -c "mysql -uroot mysql" < provision.sql +docker-compose exec -T mongo mongo < mongo-provision.js ./provision-lms.sh From bf5a63ff669e5eb65c3965e79a5e1dad26e88f20 Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Tue, 26 Oct 2021 14:07:31 -0400 Subject: [PATCH 2/7] Updated `README` to include necessary information when running multiple isolated devstacks for same host. Make additional adjustments to the documentation for multiple releases one machine. --- Makefile | 10 ++-- README.rst | 145 +++++++++++++++++++++++++++++++++++++++++++++++++++++ options.mk | 8 ++- 3 files changed, 156 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 4e6034019c..0878ff5011 100644 --- a/Makefile +++ b/Makefile @@ -217,16 +217,16 @@ xqueue_consumer-restart: ## Kill the XQueue development server. The watcher proc docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' lms-sass: ## Rebuild sass assets for the LMS container - docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=lms --theme-dirs /edx/app/edx-themes/edx-platform/ --themes my-theme' + docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=lms --theme-dirs /edx/app/edx-themes/edx-platform/' lms-static: ## Rebuild static assets for the LMS container - docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/ --themes educateworkforce greermade' + docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/' studio-sass: ## Rebuild sass assets for the Studio container - docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=cms --theme-dirs /edx/app/edx-themes/edx-platform/ --themes my-theme' + docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=cms --theme-dirs /edx/app/edx-themes/edx-platform/' -studio-static: ## Rebuild static assets for the Studio container (educateworkforce,arm,belizeproject,bmw,caregiver,dlsc,greermade,mahcc,meep,nccte,nwgrc,ptc,scelect,scjru,ts) - docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/ --themes educateworkforce greermade' +studio-static: ## Rebuild static assets for the Studio container + docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/' static: | credentials-static discovery-static ecommerce-static lms-static studio-static ## Rebuild static assets for all service containers diff --git a/README.rst b/README.rst index 42fdb5fbd3..abac66b116 100644 --- a/README.rst +++ b/README.rst @@ -404,6 +404,151 @@ images until you change or unset ``OPENEDX_RELEASE`` again. To work on the master branches and ``latest`` images, unset ``OPENEDX_RELEASE`` or set it to an empty string. +How do I run multiple named Open edX releases on same machine? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +You can have multiple isolated Devstacks provisioned on a single computer now. Follow these directions to switch between the named releases. + +#. Bring down any running containers by issuing a `make stop.all`. +#. The ``COMPOSE_PROJECT_NAME`` variable is used to define Docker namespaced volumes and network based on this value, so changing it will give you a separate set of databases. This is handled for you automatically by setting the ``OPENEDX_RELEASE`` environment variable in ``options.mk`` (e.g. ``COMPOSE_PROJECT_NAME=devstack-juniper.master``. Should you want to manually override this edit the ``options.local.mk`` in the root of this repo and create the file if it does not exist. Change the devstack project name by adding the following line: + ``COMPOSE_PROJECT_NAME=`` (e.g. ``COMPOSE_PROJECT_NAME=secondarydevstack``) +#. Perform steps in `How do I run the images for a named Open edX release?`_ for specific release. +#. Follow the steps in `Getting Started`_ section to update requirements (e.g. ``make requirements``) and provision (e.g. ``make dev.provision``) the new named release containers. + +Using with OPENEDX_RELEASE: +Many of you may use the ``OPENEDX_RELEASE`` environment variable to specify the version of named release services that are checked out during Devstack provisioning. If you set ``OPENEDX_RELEASE`` and do not specify ``COMPOSE_PROJECT_NAME``, then your Devstack project name will be set as ``devstack-${OPENEDX_RELEASE}``. + +As a specific example, if ``OPENEDX_RELEASE`` is set in your environment as ``juniper.master``, then ``COMPOSE_PROJECT_NAME`` will default to ``devstack-juniper.master`` instead of ``devstack``. + +The implication of this is that you can switch between isolated Devstack databases by changing the value of the ``OPENEDX_RELEASE`` environment variable. + +Switch between your Devstack projects by doing the following: + +#. Bring down the containers by issuing a `make stop.all`. +#. Edit the project name in ``options.local.mk`` or set ``OPENEDX_RELEASE`` and let the ``COMPOSE_PROJECT_NAME`` be assigned automatically. +#. Bring up the containers by issuing a ``make dev.up`` or ``make dev.up.all``. + +Examples of Docker Service Names After Setting the ``COMPOSE_PROJECT_NAME`` variable. Notice that the **devstack-juniper.master** name represents the ``COMPOSE_PROJECT_NAME``. + +- edx.devstack-juniper.master.lms +- edx.devstack-juniper.master.mysql + +Each instance has an isolated set of databases. This could, for example, be used to quickly switch between versions of Open edX without hitting as many issues with migrations, data integrity, etc. + +Unfortunately, this **does not** currently support running Devstacks simultaneously, because we hard-code host port numbers all over the place, and two running containers cannot share the same host port. + +Questions & Troubleshooting – Multiple Named Open edX Releases on Same Machine +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This broke my existing Devstack! +******************************** +Try posting on Discourse or Slack to see if you have the same issue as any others. If you think you have found a bug, file a CR ticket. + +I’m getting errors related to ports already being used. +******************************************************* +Make sure you bring down your devstack before changing the value of COMPOSE_PROJECT_NAME. If you forgot to, change the COMPOSE_PROJECT_NAME back to its original value, run ``make dev.down``, and then try again. + +I have custom scripts/compose files that integrate with or extend Devstack. Will those still work? +************************************************************************************************** +With the default value of COMPOSE_PROJECT_NAME = devstack, they should still work. If you choose a different COMPOSE_PROJECT_NAME, your extensions will likely break, because the names of containers change along with the project name. + +How do I enable environment variables for current directory using 'direnv'? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Recommend separating the named releases into different directories since `direenv` unloads/loads different environment variables per directory. + +#. Install `direnv` using instructions on https://direnv.net/. Below you will find additional setup at the time of this writing so refer to latest of `direnv` site for additional configuration needed. + +#. Setup the following configuration to hook `direnv` for local directory environment overrides. There are two examples for BASH or ZSH (Mac OS X) shells. + + .. code:: sh + + ## ~/.bashrc for BASH shell + + ## Hook in `direnv` for local directory environment overrides. + ## https://direnv.net/docs/hook.html + eval "$(direnv hook bash)" + + # https://github.com/direnv/direnv/wiki/Python#bash + show_virtual_env() { + if [[ -n "$VIRTUAL_ENV" && -n "$DIRENV_DIR" ]]; then + echo "($(basename $VIRTUAL_ENV))" + fi + } + export -f show_virtual_env + PS1='$(show_virtual_env)'$PS1 + + # --------------------------------------------------- + + ## ~/.zshrc for ZSH shell for Mac OS X. + + ## Hook in `direnv` for local directory environment setup. + ## https://direnv.net/docs/hook.html + eval "$(direnv hook zsh)" + + # https://github.com/direnv/direnv/wiki/Python#zsh + setopt PROMPT_SUBST + + show_virtual_env() { + if [[ -n "$VIRTUAL_ENV" && -n "$DIRENV_DIR" ]]; then + echo "($(basename $VIRTUAL_ENV))" + fi + } + PS1='$(show_virtual_env)'$PS1 + +#. Setup `layout_python-venv` function to be used in local project directory `.envrc` file. + + .. code:: sh + + ## ~/.config/direnv/direnvrc + + # https://github.com/direnv/direnv/wiki/Python#venv-stdlib-module + + realpath() { + [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}" + } + layout_python-venv() { + local python=${1:-python3} + [[ $# -gt 0 ]] && shift + unset PYTHONHOME + if [[ -n $VIRTUAL_ENV ]]; then + VIRTUAL_ENV=$(realpath "${VIRTUAL_ENV}") + else + local python_version + python_version=$("$python" -c "import platform; print(platform.python_version())") + if [[ -z $python_version ]]; then + log_error "Could not detect Python version" + return 1 + fi + VIRTUAL_ENV=$PWD/.direnv/python-venv-$python_version + fi + export VIRTUAL_ENV + if [[ ! -d $VIRTUAL_ENV ]]; then + log_status "no venv found; creating $VIRTUAL_ENV" + "$python" -m venv "$VIRTUAL_ENV" + fi + + PATH="${VIRTUAL_ENV}/bin:${PATH}" + export PATH + } + +#. Example `.envrc` file used in project directory. Need to make sure that each release root has this unique file. + + .. code:: sh + + # Open edX named release project directory root. + ## /devstack.juniper/.envrc + + # https://discuss.openedx.org/t/docker-devstack-multiple-releases-one-machine/1902/10 + + # This is handled when OPENEDX_RELEASE is set. Leaving this in for manual override. + # export COMPOSE_PROJECT_NAME=devstack-juniper + + export DEVSTACK_WORKSPACE="$(pwd)" + export OPENEDX_RELEASE=juniper.master + export VIRTUAL_ENV="$(pwd)/devstack/venv" + + # https://github.com/direnv/direnv/wiki/Python#virtualenv + layout python-venv + How do I create database dumps? ------------------------------- We use database dumps to speed up provisioning and generally spend less time running migrations. These dumps should be diff --git a/options.mk b/options.mk index 0e0a456099..62a16f0afc 100644 --- a/options.mk +++ b/options.mk @@ -9,8 +9,12 @@ DEVSTACK_WORKSPACE ?= $(shell pwd)/.. # Name of Docker Compose project. -# See https://docs.docker.com/compose/reference/envvars/#compose_project_name -# Defaults to 'devstack' should OPENEDX_RELEASE not be defined. +# Volumes and network are namespaced based on this value, +# so changing it will give you a separate set of databases. +# See https://docs.docker.com/compose/reference/envvars/#compose_project_name +# If OPENEDX_RELAESE is defined, defaults to `devstack-${OPENEDX_RELEASE}`; +# otherwise, it defaults to `devstack`. +# Be sure to bring down services before changing the value of `COMPOSE_PROJECT_NAME`. ifdef OPENEDX_RELEASE COMPOSE_PROJECT_NAME ?= devstack-${OPENEDX_RELEASE} else From e8373529f459a125e32330b043eef9a13f0c681e Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Tue, 26 Oct 2021 14:16:00 -0400 Subject: [PATCH 3/7] Make sure to pass in `DOCKER_COMPOSE_FILES` to `docker-compose exec` commands for services outside of `docker-compose.yml`. Makes sure that other services (e.g. `xqueue`, etc) that are not defined in the docker-compose.yml file have the correct configuration passed to the `docker-compose exec` command to work properly. --- Makefile | 60 +++++++++++++++--------------- Makefile.edx | 4 +- course-generator/create-courses.sh | 12 +++--- marketing.mk | 2 +- programs/provision.sh | 2 +- provision-analytics-pipeline.sh | 14 ++++--- provision-credentials.sh | 10 ++--- provision-discovery.sh | 8 ++-- provision-e2e.sh | 8 ++-- provision-ecommerce.sh | 6 +-- provision-forum.sh | 2 +- provision-ida-user.sh | 4 +- provision-ida.sh | 8 ++-- provision-lms.sh | 16 ++++---- provision-notes.sh | 2 +- provision-retirement-user.sh | 4 +- provision.sh | 6 +-- 17 files changed, 86 insertions(+), 82 deletions(-) diff --git a/Makefile b/Makefile index 0878ff5011..fa074707b4 100644 --- a/Makefile +++ b/Makefile @@ -83,7 +83,7 @@ dev.up.all: | dev.up dev.up.watchers ## Bring up all services with host volumes, dev.print-container.%: ## Get the ID of the running container for a given service. @# Can be run as ``make --silent dev.print-container.$service`` for just ID. - @echo $$(docker-compose ps --quiet $*) + @echo $$(docker-compose $(DOCKER_COMPOSE_FILES) ps --quiet $*) dev.sync.daemon.start: ## Start the docker-sycn daemon docker-sync start @@ -101,7 +101,7 @@ provision: | dev.provision ## This command will be deprecated in a future releas stop: ## Stop all services (test -d .docker-sync && docker-sync stop) || true ## Ignore failure here - docker-compose stop + docker-compose $(DOCKER_COMPOSE_FILES) stop stop.watchers: ## Stop asset watchers docker-compose -f docker-compose-watchers.yml stop @@ -152,36 +152,36 @@ restore: ## Restore all data volumes from the host. WARNING: THIS WILL OVERWRIT # TODO: Print out help for this target. Even better if we can iterate over the # services in docker-compose.yml, and print the actual service names. %-shell: ## Run a shell on the specified service container - docker-compose exec $* /bin/bash + docker-compose $(DOCKER_COMPOSE_FILES) exec $* /bin/bash credentials-shell: ## Run a shell on the credentials container - docker-compose exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' + docker-compose $(DOCKER_COMPOSE_FILES) exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' discovery-shell: ## Run a shell on the discovery container - docker-compose exec discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open ecommerce-shell: ## Run a shell on the ecommerce container - docker-compose exec ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open e2e-shell: ## Start the end-to-end tests container with a shell docker-compose run --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash %-update-db: ## Run migrations for the specified service container - docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' + docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' studio-update-db: ## Run migrations for the Studio container - docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' + docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' lms-update-db: ## Run migrations LMS container - docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' + docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' update-db: | studio-update-db lms-update-db discovery-update-db ecommerce-update-db credentials-update-db ## Run the migrations for all services lms-shell: ## Run a shell on the LMS container - docker-compose exec lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open lms-watcher-shell: ## Run a shell on the LMS watcher container - docker-compose exec lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open %-attach: ## Attach to the specified service container process to use the debugger & see logs. docker attach "$$(make --silent dev.print-container.$*)" @@ -190,43 +190,43 @@ lms-restart: ## Kill the LMS Django development server. The watcher process will docker-compose exec lms bash -c 'kill $$(ps aux | grep "manage.py lms" | egrep -v "while|grep" | awk "{print \$$2}")' studio-shell: ## Run a shell on the Studio container - docker-compose exec studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-watcher-shell: ## Run a shell on the studio watcher container - docker-compose exec studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-restart: ## Kill the LMS Django development server. The watcher process will restart it. docker-compose exec studio bash -c 'kill $$(ps aux | grep "manage.py cms" | egrep -v "while|grep" | awk "{print \$$2}")' xqueue-shell: ## Run a shell on the XQueue container - docker-compose exec xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue-restart: ## Kill the XQueue development server. The watcher process will restart it. docker-compose exec xqueue bash -c 'kill $$(ps aux | grep "manage.py runserver" | egrep -v "while|grep" | awk "{print \$$2}")' %-restart-devserver: ## Kill a service's Django development server. The watcher process should restart it. - docker-compose exec $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' xqueue_consumer-shell: ## Run a shell on the XQueue consumer container - docker-compose exec xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue_consumer-restart: ## Kill the XQueue development server. The watcher process will restart it. - docker-compose exec xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")' %-static: ## Rebuild static assets for the specified service container - docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' + docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' lms-sass: ## Rebuild sass assets for the LMS container - docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=lms --theme-dirs /edx/app/edx-themes/edx-platform/' + docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=lms --theme-dirs /edx/app/edx-themes/edx-platform/' lms-static: ## Rebuild static assets for the LMS container - docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/' + docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/' studio-sass: ## Rebuild sass assets for the Studio container - docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=cms --theme-dirs /edx/app/edx-themes/edx-platform/' + docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=cms --theme-dirs /edx/app/edx-themes/edx-platform/' studio-static: ## Rebuild static assets for the Studio container - docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/' + docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/' static: | credentials-static discovery-static ecommerce-static lms-static studio-static ## Rebuild static assets for all service containers @@ -238,24 +238,24 @@ e2e-tests: ## Run the end-to-end tests against the service containers validate-lms-volume: ## Validate that changes to the local workspace are reflected in the LMS container touch $(DEVSTACK_WORKSPACE)/edx-platform/testfile - docker-compose exec -T lms ls /edx/app/edxapp/edx-platform/testfile + docker-compose $(DOCKER_COMPOSE_FILES) exec -T lms ls /edx/app/edxapp/edx-platform/testfile rm $(DEVSTACK_WORKSPACE)/edx-platform/testfile vnc-passwords: ## Get the VNC passwords for the Chrome and Firefox Selenium containers - @docker-compose logs chrome 2>&1 | grep "VNC password" | tail -1 - @docker-compose logs firefox 2>&1 | grep "VNC password" | tail -1 + @docker-compose $(DOCKER_COMPOSE_FILES) logs chrome 2>&1 | grep "VNC password" | tail -1 + @docker-compose $(DOCKER_COMPOSE_FILES) logs firefox 2>&1 | grep "VNC password" | tail -1 devpi-password: ## Get the root devpi password for the devpi container - docker-compose exec devpi bash -c "cat /data/server/.serverpassword" + docker-compose $(DOCKER_COMPOSE_FILES) exec devpi bash -c "cat /data/server/.serverpassword" mysql-shell: ## Run a shell on the mysql container - docker-compose exec mysql bash + docker-compose $(DOCKER_COMPOSE_FILES) exec mysql bash mysql-shell-edxapp: ## Run a mysql shell on the edxapp database - docker-compose exec mysql bash -c "mysql edxapp" + docker-compose $(DOCKER_COMPOSE_FILES) exec mysql bash -c "mysql edxapp" mongo-shell: ## Run a shell on the mongo container - docker-compose exec mongo bash + docker-compose $(DOCKER_COMPOSE_FILES) exec mongo bash ### analytics pipeline commands @@ -281,7 +281,7 @@ stop.analytics_pipeline: ## Stop analytics pipeline services docker-compose up -d mysql ## restart mysql as other containers need it hadoop-application-logs-%: ## View hadoop logs by application Id - docker-compose exec analytics_pipeline.nodemanager yarn logs -applicationId $* + docker-compose $(DOCKER_COMPOSE_FILES) exec analytics_pipeline.nodemanager yarn logs -applicationId $* # Provisions studio, ecommerce, and marketing with course(s) in test-course.json # Modify test-course.json before running this make target to generate a custom course diff --git a/Makefile.edx b/Makefile.edx index 9ce4a6c1d3..fb75ff614e 100644 --- a/Makefile.edx +++ b/Makefile.edx @@ -25,10 +25,10 @@ dev.provision.whitelabel: whitelabel-tests: docker-compose run -T -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh $(make --silent dev.print-container.devstack.whitelabel)":/tmp/run_whitelabel_tests.sh - docker-compose exec devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh + docker-compose $(DOCKER_COMPOSE_FILES) exec devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh whitelabel-cleanup: docker-compose rm --force devstack.whitelabel whitelabel-shell: ## Start a whitelabel test shell session - docker-compose exec -it devstack.whitelabel env TERM=$(TERM) bash + docker-compose $(DOCKER_COMPOSE_FILES) exec -it devstack.whitelabel env TERM=$(TERM) bash diff --git a/course-generator/create-courses.sh b/course-generator/create-courses.sh index b7ee059a9e..54e2bc3629 100755 --- a/course-generator/create-courses.sh +++ b/course-generator/create-courses.sh @@ -8,21 +8,21 @@ echo "Parsing options" container_error=false for arg in "$@"; do if [ $arg == "--studio" ]; then - if [ ! "$(docker-compose exec lms bash -c 'echo "Course will be created for studio"; exit $?')" ]; then + if [ ! "$(docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'echo "Course will be created for studio"; exit $?')" ]; then echo "Issue with studio container" container_error=true else studio=true fi elif [ $arg == "--ecommerce" ]; then - if [ ! "$(docker-compose exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then + if [ ! "$(docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then echo "Issue with ecommerce container" container_error=true else ecommerce=true fi elif [ $arg == "--marketing" ]; then - if [ ! "$(docker-compose exec marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then + if [ ! "$(docker-compose $DOCKER_COMPOSE_FILES exec marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then echo "Issue with marketing container. Course creation will proceed without marketing container." else marketing=true @@ -49,15 +49,15 @@ done < "${@: -1}" if $studio ; then echo "Creating courses on studio." - docker-compose exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'" + docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'" fi if $ecommerce ; then echo "Creating courses on ecommerce." - docker-compose exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" + docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" fi if $marketing ; then echo "Creating courses on marketing." - docker-compose exec marketing bash -c "drush generate_courses '$course_json'" + docker-compose $DOCKER_COMPOSE_FILES exec marketing bash -c "drush generate_courses '$course_json'" fi diff --git a/marketing.mk b/marketing.mk index 765a47adc5..349872d450 100644 --- a/marketing.mk +++ b/marketing.mk @@ -3,7 +3,7 @@ help-marketing: ## Display this help message @perl -nle'print $& if m{^[\.a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | grep marketing | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m %-25s\033[0m %s\n", $$1, $$2}' marketing-shell: ## Run a shell on the marketing site container - docker-compose exec marketing env TERM=$(TERM) bash -c 'cd /edx/app/edx-mktg/edx-mktg; exec /bin/bash -sh' + docker-compose $(DOCKER_COMPOSE_FILES) exec marketing env TERM=$(TERM) bash -c 'cd /edx/app/edx-mktg/edx-mktg; exec /bin/bash -sh' stop-marketing: ## Stop all services (including the marketing site) with host volumes docker-compose -f docker-compose.yml -f docker-compose-host.yml -f docker-compose-marketing-site.yml -f docker-compose-marketing-site-host.yml stop diff --git a/programs/provision.sh b/programs/provision.sh index 8ce3d1567a..939a719603 100755 --- a/programs/provision.sh +++ b/programs/provision.sh @@ -42,7 +42,7 @@ docker_exec() { /edx/app/$app/$repo/manage.py $cmd " - docker-compose exec "$service" bash -c "$CMDS" + docker-compose $DOCKER_COMPOSE_FILES exec -T "$service" bash -c "$CMDS" } provision_ida() { diff --git a/provision-analytics-pipeline.sh b/provision-analytics-pipeline.sh index 3f00f1892c..1798fc334f 100755 --- a/provision-analytics-pipeline.sh +++ b/provision-analytics-pipeline.sh @@ -34,24 +34,24 @@ sleep 20 # Analytics pipeline has dependency on lms but we only need its db schema & not full lms. So we'll just load their db # schemas as part of analytics pipeline provisioning. If there is a need of a fully fledged LMS, then provision lms # by following their documentation. -if [[ ! -z "`docker-compose exec mysql mysql -uroot -se "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='edxapp'" 2>&1`" ]]; +if [[ ! -z "`docker-compose $DOCKER_COMPOSE_FILES exec mysql mysql -uroot -se "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='edxapp'" 2>&1`" ]]; then echo -e "${GREEN}LMS DB exists, skipping lms schema load.${NC}" else echo -e "${GREEN}LMS DB not found, provisioning lms schema.${NC}" - docker-compose exec mysql bash -c 'mysql -uroot mysql' < provision.sql + docker-compose $DOCKER_COMPOSE_FILES exec mysql bash -c 'mysql -uroot mysql' < provision.sql ./load-db.sh edxapp docker-compose $DOCKER_COMPOSE_FILES up -d lms studio - docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' + docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' #Installing prereqs crashes the process docker-compose restart lms # Run edxapp migrations first since they are needed for the service users and OAuth clients - docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' + docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' fi echo -e "${GREEN}LMS database provisioned successfully...${NC}" echo -e "${GREEN}Creating databases and users...${NC}" -docker-compose exec mysql bash -c 'mysql -uroot mysql' < provision-analytics-pipeline.sql +docker-compose $DOCKER_COMPOSE_FILES exec mysql bash -c 'mysql -uroot mysql' < provision-analytics-pipeline.sql # initialize hive metastore echo -e "${GREEN}Initializing HIVE metastore...${NC}" @@ -68,8 +68,12 @@ done sleep 10 # for datanode & other services to activate echo -e "${GREEN}Namenode is ready!${NC}" +<<<<<<< HEAD:provision-analytics-pipeline.sh docker-compose exec -u hadoop analytics_pipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' docker image prune -f echo -e "${GREEN}Analytics pipeline provisioning complete!${NC}" +======= +docker-compose $DOCKER_COMPOSE_FILES exec -u hadoop analytics_pipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' +>>>>>>> 357f8a8... Make sure to pass in `DOCKER_COMPOSE_FILES` to `docker-compose exec` commands for services outside of `docker-compose.yml`.:provision-analyticspipeline.sh diff --git a/provision-credentials.sh b/provision-credentials.sh index 0334f2966b..1029ce339d 100755 --- a/provision-credentials.sh +++ b/provision-credentials.sh @@ -9,20 +9,20 @@ port=18150 docker-compose $DOCKER_COMPOSE_FILES up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" echo -e "${GREEN}Configuring site for ${name}...${NC}" -docker-compose exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker-compose exec ${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-discovery.sh b/provision-discovery.sh index 2ffec8d1d2..d58f79351e 100755 --- a/provision-discovery.sh +++ b/provision-discovery.sh @@ -1,10 +1,10 @@ # Provisioning script for the discovery service ./provision-ida.sh discovery discovery 18381 -docker-compose exec discovery bash -c 'rm -rf /edx/var/discovery/*' -docker-compose exec discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 1 --site-domain localhost:18381 --code edx --name edX --courses-api-url "http://edx.devstack.lms:18000/api/courses/v1/" --ecommerce-api-url "http://edx.devstack.ecommerce:18130/api/v2/" --organizations-api-url "http://edx.devstack.lms:18000/api/organizations/v0/" --oidc-url-root "http://edx.devstack.lms:18000/oauth2" --oidc-key discovery-key --oidc-secret discovery-secret' -docker-compose exec discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py refresh_course_metadata' -docker-compose exec discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py update_index --disable-change-limit' +docker-compose $DOCKER_COMPOSE_FILES exec -T discovery bash -c 'rm -rf /edx/var/discovery/*' +docker-compose $DOCKER_COMPOSE_FILES exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 1 --site-domain localhost:18381 --code edx --name edX --courses-api-url "http://edx.devstack.lms:18000/api/courses/v1/" --ecommerce-api-url "http://edx.devstack.ecommerce:18130/api/v2/" --organizations-api-url "http://edx.devstack.lms:18000/api/organizations/v0/" --lms-url "http://edx.devstack.lms:18000/"' +docker-compose $DOCKER_COMPOSE_FILES exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py refresh_course_metadata' +docker-compose $DOCKER_COMPOSE_FILES exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py update_index --disable-change-limit' # Add demo program ./programs/provision.sh discovery diff --git a/provision-e2e.sh b/provision-e2e.sh index caadd418f5..baee623cd0 100755 --- a/provision-e2e.sh +++ b/provision-e2e.sh @@ -13,11 +13,11 @@ fi docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz "$(make --silent dev.print-container.studio)":/tmp/ # Extract the test course tarball -docker-compose exec studio bash -c 'cd /tmp && tar xzf course.tar.gz' +docker-compose $DOCKER_COMPOSE_FILES exec -T studio bash -c 'cd /tmp && tar xzf course.tar.gz' # Import the course content -docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker import /tmp course' +docker-compose $DOCKER_COMPOSE_FILES exec -T studio bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker import /tmp course' # Clean up the temp files -docker-compose exec studio bash -c 'rm /tmp/course.tar.gz' -docker-compose exec studio bash -c 'rm -r /tmp/course' +docker-compose $DOCKER_COMPOSE_FILES exec -T studio bash -c 'rm /tmp/course.tar.gz' +docker-compose $DOCKER_COMPOSE_FILES exec -T studio bash -c 'rm -r /tmp/course' diff --git a/provision-ecommerce.sh b/provision-ecommerce.sh index 00068b6c0e..3846ad7045 100755 --- a/provision-ecommerce.sh +++ b/provision-ecommerce.sh @@ -4,6 +4,6 @@ ./provision-ida.sh ecommerce ecommerce 18130 # Configure ecommerce -docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --client-id=ecommerce-key --client-secret=ecommerce-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --journals_api_url=http://journals.app:18606/api/v1/' -docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' -docker-compose exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' +docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --client-id=ecommerce-key --client-secret=ecommerce-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --journals_api_url=http://journals.app:18606/api/v1/' +docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' +docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' diff --git a/provision-forum.sh b/provision-forum.sh index 793a799b73..8e1f13b864 100755 --- a/provision-forum.sh +++ b/provision-forum.sh @@ -3,4 +3,4 @@ set -o pipefail set -x docker-compose $DOCKER_COMPOSE_FILES up -d forum -docker-compose exec forum bash -c 'source /edx/app/forum/ruby_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/' +docker-compose $DOCKER_COMPOSE_FILES exec -T forum bash -c 'source /edx/app/forum/ruby_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/' diff --git a/provision-ida-user.sh b/provision-ida-user.sh index 57460b590c..a7fc91572b 100755 --- a/provision-ida-user.sh +++ b/provision-ida-user.sh @@ -5,5 +5,5 @@ client_name=$2 client_port=$3 echo -e "${GREEN}Creating service user and OAuth client for ${app_name}...${NC}" -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_oauth2_client "http://localhost:$3" "http://localhost:$3/complete/edx-oidc/" confidential --client_name $2 --client_id "$1-key" --client_secret "$1-secret" --trusted --logout_uri "http://localhost:$3/logout/" --username $1_worker' -- "$app_name" "$client_name" "$client_port" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_oauth2_client "http://localhost:$3" "http://localhost:$3/complete/edx-oidc/" confidential --client_name $2 --client_id "$1-key" --client_secret "$1-secret" --trusted --logout_uri "http://localhost:$3/logout/" --username $1_worker' -- "$app_name" "$client_name" "$client_port" diff --git a/provision-ida.sh b/provision-ida.sh index a0b38ab5ce..34e80ead6a 100755 --- a/provision-ida.sh +++ b/provision-ida.sh @@ -5,17 +5,17 @@ client_port=$3 # The port corresponding to this IDA service in devstack. docker-compose $DOCKER_COMPOSE_FILES up -d $app_name echo -e "${GREEN}Installing requirements for ${app_name}...${NC}" -docker-compose exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" echo -e "${GREEN}Running migrations for ${app_name}...${NC}" -docker-compose exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" echo -e "${GREEN}Creating super-user for ${app_name}...${NC}" -docker-compose exec ${app_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec ${app_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" ./provision-ida-user.sh $app_name $client_name $client_port # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${app_name}...${NC}" -docker-compose exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" diff --git a/provision-lms.sh b/provision-lms.sh index 3b7439a06d..991b04e9bb 100755 --- a/provision-lms.sh +++ b/provision-lms.sh @@ -13,33 +13,33 @@ for app in "${apps[@]}"; do docker-compose $DOCKER_COMPOSE_FILES up -d $app done -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' #Installing prereqs crashes the process docker-compose restart lms # Run edxapp migrations first since they are needed for the service users and OAuth clients -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' # Create a superuser for edxapp -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user edx edx@example.com --superuser --staff' -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username=\"edx\"); user.set_password(\"edx\"); user.save()" | python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user edx edx@example.com --superuser --staff' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username=\"edx\"); user.set_password(\"edx\"); user.save()" | python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' # Create an enterprise service user for edxapp docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user enterprise_worker enterprise_worker@example.com' # Enable the LMS-E-Commerce integration -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce' # Create demo course and users -docker-compose exec lms bash -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"' # Fix missing vendor file by clearing the cache -docker-compose exec lms bash -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1' # Create static assets for both LMS and Studio for app in "${apps[@]}"; do - docker-compose exec $app bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker' + docker-compose $DOCKER_COMPOSE_FILES exec -T $app bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker' done # Provision a retirement service account user diff --git a/provision-notes.sh b/provision-notes.sh index a84c077a22..9d701ae351 100755 --- a/provision-notes.sh +++ b/provision-notes.sh @@ -5,4 +5,4 @@ # This will build the elasticsearch index for notes. echo -e "${GREEN}Creating indexes for edx_notes_api...${NC}" -docker-compose exec edx_notes_api bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py rebuild_index --noinput' -- edx_notes_api +docker-compose $DOCKER_COMPOSE_FILES exec edx_notes_api bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py rebuild_index --noinput' -- edx_notes_api diff --git a/provision-retirement-user.sh b/provision-retirement-user.sh index 3267258628..9a9f209c6c 100755 --- a/provision-retirement-user.sh +++ b/provision-retirement-user.sh @@ -4,5 +4,5 @@ app_name=$1 user_name=$2 echo -e "${GREEN}Creating retirement service user ${user_name} and DOT Application ${app_name}...${NC}" -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" diff --git a/provision.sh b/provision.sh index adaff4246e..118b30f935 100755 --- a/provision.sh +++ b/provision.sh @@ -22,7 +22,7 @@ docker-compose up -d mysql mongo # Ensure the MySQL server is online and usable echo "${GREEN}Waiting for MySQL.${NC}" -until docker-compose exec -T mysql bash -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null +until docker-compose $DOCKER_COMPOSE_FILES exec -T mysql bash -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 @@ -35,8 +35,8 @@ sleep 20 # Ensure that the MySQL databases and users are created for all IDAs. # (A no-op for databases and users that already exist). echo -e "${GREEN}Ensuring MySQL databases and users exist...${NC}" -docker-compose exec -T mysql bash -c "mysql -uroot mysql" < provision.sql -docker-compose exec -T mongo mongo < mongo-provision.js +docker-compose $DOCKER_COMPOSE_FILES exec -T mysql bash -c "mysql -uroot mysql" < provision.sql +docker-compose $DOCKER_COMPOSE_FILES exec -T mongo mongo < mongo-provision.js ./provision-lms.sh From 9e79f072d68f11d25cb0302f3bfea23e4a25d9bd Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Mon, 10 Aug 2020 16:32:08 -0400 Subject: [PATCH 4/7] Add `xqueue` to default services to provision initially. --- options.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options.mk b/options.mk index 62a16f0afc..b09035515b 100644 --- a/options.mk +++ b/options.mk @@ -47,7 +47,7 @@ FS_SYNC_STRATEGY ?= local-mounts # TODO: Re-evaluate this list and consider paring it down to a tighter core. # The current value was chosen such that it would not change the existing # Devstack behavior. -DEFAULT_SERVICES ?= lms+studio+ecommerce+discovery+credentials+forum+edx_notes_api+gradebook+frontend-app-publisher +DEFAULT_SERVICES ?= lms+studio+ecommerce+discovery+xqueue+credentials+forum+edx_notes_api+gradebook+frontend-app-publisher # List of all services with database migrations. # Services must provide a Makefile target named: $(service)-update-db From 2be62b4d312c1037246c28e4ab5091974a967a1e Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Tue, 26 Oct 2021 14:19:03 -0400 Subject: [PATCH 5/7] Make additional adjustments to the documentation for multiple releases one machine (regisb review). Comments came from `master` https://github.com/edx/devstack/pull/569/commits/c0b482d9ff1355b3d59ee25b63b3ca79d93ff372 Fix issue with `docker-compose config` not outputting configuration information when `$(DOCKER_COMPOSE_FILES)` was excluded from the command. It appears that leaving out `$(DOCKER_COMPOSE_FILES)` caused the `docker-compose config` to not include the **volume** information for services within the `docker-compose.yml` file, as well as, excluding additional services like `docker-compose-{analytics-pipeline,watchers}.yml` to name a few. Update provision-ecommerce.sh with recommended changes from @morenol Co-authored-by: morenol <13-10934@usb.ve> Merging additional updates based on #569 changes for the `readme` file. - All changes from https://github.com/edx/devstack/pull/569 changes from this. --- Makefile | 2 +- README.rst | 58 ++++++++++++++++++++++++++++++------------ provision-ecommerce.sh | 2 +- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index fa074707b4..1d37b4c6f7 100644 --- a/Makefile +++ b/Makefile @@ -137,7 +137,7 @@ pull.xqueue: ## Update XQueue Docker images docker-compose -f docker-compose-xqueue.yml pull --parallel validate: ## Validate the devstack configuration - docker-compose config + docker-compose $(DOCKER_COMPOSE_FILES) config backup: ## Write all data volumes to the host. docker run --rm --volumes-from -T $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql diff --git a/README.rst b/README.rst index abac66b116..a9f7149a10 100644 --- a/README.rst +++ b/README.rst @@ -405,7 +405,7 @@ master branches and ``latest`` images, unset ``OPENEDX_RELEASE`` or set it to an empty string. How do I run multiple named Open edX releases on same machine? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------------------------------- You can have multiple isolated Devstacks provisioned on a single computer now. Follow these directions to switch between the named releases. #. Bring down any running containers by issuing a `make stop.all`. @@ -414,18 +414,19 @@ You can have multiple isolated Devstacks provisioned on a single computer now. F #. Perform steps in `How do I run the images for a named Open edX release?`_ for specific release. #. Follow the steps in `Getting Started`_ section to update requirements (e.g. ``make requirements``) and provision (e.g. ``make dev.provision``) the new named release containers. -Using with OPENEDX_RELEASE: -Many of you may use the ``OPENEDX_RELEASE`` environment variable to specify the version of named release services that are checked out during Devstack provisioning. If you set ``OPENEDX_RELEASE`` and do not specify ``COMPOSE_PROJECT_NAME``, then your Devstack project name will be set as ``devstack-${OPENEDX_RELEASE}``. - As a specific example, if ``OPENEDX_RELEASE`` is set in your environment as ``juniper.master``, then ``COMPOSE_PROJECT_NAME`` will default to ``devstack-juniper.master`` instead of ``devstack``. The implication of this is that you can switch between isolated Devstack databases by changing the value of the ``OPENEDX_RELEASE`` environment variable. -Switch between your Devstack projects by doing the following: +Switch between your Devstack releases by doing the following: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +#. Bring down the containers by issuing a ``make stop.all`` for the running release. +#. Follow the instructions from the `How do I run multiple named Open edX releases on same machine?`_ section. +#. Edit the project name in ``options.local.mk`` or set the ``OPENEDX_RELEASE`` environment variable and let the ``COMPOSE_PROJECT_NAME`` be assigned automatically. +#. Bring up the containers with ``make dev.up``. -#. Bring down the containers by issuing a `make stop.all`. -#. Edit the project name in ``options.local.mk`` or set ``OPENEDX_RELEASE`` and let the ``COMPOSE_PROJECT_NAME`` be assigned automatically. -#. Bring up the containers by issuing a ``make dev.up`` or ``make dev.up.all``. +**NOTE:** Additional instructions on switching releases using `direnv` can be found in `How do I switch releases using 'direnv'?`_ section. Examples of Docker Service Names After Setting the ``COMPOSE_PROJECT_NAME`` variable. Notice that the **devstack-juniper.master** name represents the ``COMPOSE_PROJECT_NAME``. @@ -437,23 +438,48 @@ Each instance has an isolated set of databases. This could, for example, be used Unfortunately, this **does not** currently support running Devstacks simultaneously, because we hard-code host port numbers all over the place, and two running containers cannot share the same host port. Questions & Troubleshooting – Multiple Named Open edX Releases on Same Machine -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------------------------------------------ This broke my existing Devstack! -******************************** -Try posting on Discourse or Slack to see if you have the same issue as any others. If you think you have found a bug, file a CR ticket. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + See if the troubleshooting of this readme can help resolve your broken devstack first, then try posting on the `Open edX forums `__ to see if you have the same issue as any others. If you think you have found a bug, file a CR ticket. I’m getting errors related to ports already being used. -******************************************************* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Make sure you bring down your devstack before changing the value of COMPOSE_PROJECT_NAME. If you forgot to, change the COMPOSE_PROJECT_NAME back to its original value, run ``make dev.down``, and then try again. I have custom scripts/compose files that integrate with or extend Devstack. Will those still work? -************************************************************************************************** +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ With the default value of COMPOSE_PROJECT_NAME = devstack, they should still work. If you choose a different COMPOSE_PROJECT_NAME, your extensions will likely break, because the names of containers change along with the project name. - + +How do I switch releases using 'direnv'? +---------------------------------------- + +Follow directions in `Switch between your Devstack releases by doing the following:`_ then make the following adjustments. + +Make sure that you have setup each Open edX release in separate directories using `How do I enable environment variables for current directory using 'direnv'?`_ instructions. Open the next release project in a separate code editor, then activate the ``direnv`` environment variables and virtual environment for the next release by using a terminal shell to traverse to the directory with the corresponding release ``.envrc`` file. You may need to issue a ``direnv allow`` command to enable the ``.envrc`` file. + + .. code:: sh + + # You should see something like the following after successfully enabling 'direnv' for the Juniper release. + + direnv: loading ~/open-edx/devstack.juniper/.envrc + direnv: export +DEVSTACK_WORKSPACE +OPENEDX_RELEASE +VIRTUAL_ENV ~PATH + (venv)username@computer-name devstack.juniper % + +**NOTE:** Setting of the ``OPENEDX_RELEASE`` should have been handled within the ``.envrc`` file for named releases only and should not be defined for the ``master`` release. + How do I enable environment variables for current directory using 'direnv'? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Recommend separating the named releases into different directories since `direenv` unloads/loads different environment variables per directory. +--------------------------------------------------------------------------- +We recommend separating the named releases into different directories, for clarity purposes. You can use `direnv `__ to define different environment variables per directory:: + + .. code:: + + # Example showing directory structure for separate Open edX releases. + + /Users//open-edx – root directory for platform development + |_ ./devstack.master – directory containing all repository information related to the main development release. + |_ ./devstack.juniper – directory containing all repository information related to the Juniper release. #. Install `direnv` using instructions on https://direnv.net/. Below you will find additional setup at the time of this writing so refer to latest of `direnv` site for additional configuration needed. diff --git a/provision-ecommerce.sh b/provision-ecommerce.sh index 3846ad7045..bfefe42217 100755 --- a/provision-ecommerce.sh +++ b/provision-ecommerce.sh @@ -4,6 +4,6 @@ ./provision-ida.sh ecommerce ecommerce 18130 # Configure ecommerce -docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --client-id=ecommerce-key --client-secret=ecommerce-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --journals_api_url=http://journals.app:18606/api/v1/' +docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/' docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' From 4d476e6818aa44dfb6d6754714d9e6b33df44c7d Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Tue, 26 Oct 2021 14:26:15 -0400 Subject: [PATCH 6/7] Removing additional changes from Juniper release while merging to Hawthorn. --- docker-compose-watchers-nfs.yml | 34 ---------------------- docker-compose.yml | 51 --------------------------------- 2 files changed, 85 deletions(-) delete mode 100644 docker-compose-watchers-nfs.yml diff --git a/docker-compose-watchers-nfs.yml b/docker-compose-watchers-nfs.yml deleted file mode 100644 index 532a49060f..0000000000 --- a/docker-compose-watchers-nfs.yml +++ /dev/null @@ -1,34 +0,0 @@ -version: "2.1" - -services: - lms_watcher: - command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.lms_watcher - environment: - BOK_CHOY_HOSTNAME: edx.devstack.lms_watcher - ASSET_WATCHER_TIMEOUT: 12 - image: edxops/edxapp:${OPENEDX_RELEASE:-latest} - volumes: - - edx-nfs:/edx/app/edxapp/edx-platform - - edxapp_lms_assets:/edx/var/edxapp/staticfiles/ - - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules - - src-nfs:/edx/src - - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes:cached - - studio_watcher: - command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done' - container_name: edx.devstack.studio_watcher - environment: - BOK_CHOY_HOSTNAME: edx.devstack.studio_watcher - ASSET_WATCHER_TIMEOUT: 12 - image: edxops/edxapp:${OPENEDX_RELEASE:-latest} - volumes: - - edxapp_studio_assets:/edx/var/edxapp/staticfiles/ - - edx-nfs:/edx/app/edxapp/edx-platform - - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules - - src-nfs:/edx/src:cached - - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes:cached -volumes: - edxapp_lms_assets: - edxapp_studio_assets: - edxapp_node_modules: diff --git a/docker-compose.yml b/docker-compose.yml index edf0277105..9814a3609e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,12 +43,6 @@ services: - elasticsearch_data:/usr/share/elasticsearch/data - elasticsearch_data:/usr/share/elasticsearch/logs - # This is meant to be used to test ES upgrades so that we do not have to upgrade all of our services to ES5 at once. - elasticsearch-5: - container_name: edx.devstack.elasticsearch-5 - hostname: elasticsearch-5.devstack.edx - image: elasticsearch:5.6.16 - firefox: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.firefox" hostname: firefox.devstack.edx @@ -109,12 +103,6 @@ services: volumes: - mysql_data:/var/lib/mysql - redis: - container_name: edx.devstack.redis - hostname: redis.devstack.edx - image: redis:2.8 - command: redis-server --requirepass password - # ================================================ # edX services credentials: @@ -288,34 +276,6 @@ services: volumes: - edxapp_studio_assets:/edx/var/edxapp/staticfiles/ - # ========================================================================== - # edX Microfrontends - # - # TODO: Instead of having an nginx container for every single microfrontend - # (there are like 12 now, right?), we should come up with an actual strategy - # for micro-frontends in devtack. - # ========================================================================== - - frontend-app-publisher: - extends: - file: microfrontend.yml - service: microfrontend - working_dir: '/edx/app/frontend-app-publisher' - container_name: edx.devstack.frontend-app-publisher - ports: - - "18400:18400" - depends_on: - - lms - - gradebook: - extends: - file: microfrontend.yml - service: microfrontend - working_dir: '/edx/app/gradebook' - container_name: edx.devstack.gradebook - ports: - - "44567:4567" - devpi: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.devpi" hostname: devpi.devstack.edx @@ -329,17 +289,6 @@ services: volumes: - devpi_data:/data - frontend-app-learning: - extends: - file: microfrontend.yml - service: microfrontend - working_dir: '/edx/app/frontend-app-learning' - container_name: edx.devstack.frontend-app-learning - ports: - - "2000:2000" - depends_on: - - lms - volumes: discovery_assets: edxapp_lms_assets: From 33407f2f9baca1eb32f92b597d425a2b53de8d80 Mon Sep 17 00:00:00 2001 From: Zachary Trabookis Date: Tue, 26 Oct 2021 19:31:40 -0400 Subject: [PATCH 7/7] Additional changes to make work with the Hawthorn release. --- Makefile | 94 ++++++++++++++++++++++++++++++++++++++++++++---------- options.mk | 6 ++-- 2 files changed, 81 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 1d37b4c6f7..3d7af06317 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,35 @@ include options.mk # Include local makefile with additional targets. -include local.mk # Prefix with hyphen to tolerate absence of file. +ifeq ($(FS_SYNC_STRATEGY),local-mounts) +DOCKER_COMPOSE_FILES := \ +-f docker-compose-host.yml \ +-f docker-compose-themes.yml \ +-f docker-compose-watchers.yml \ +-f docker-compose-xqueue.yml \ +-f docker-compose-analytics-pipeline.yml \ +-f docker-compose-marketing-site.yml +endif + +# ifeq ($(FS_SYNC_STRATEGY),nfs) +# DOCKER_COMPOSE_FILES := \ +# -f docker-compose-host-nfs.yml \ +# -f docker-compose-themes-nfs.yml \ +# -f docker-compose-watchers-nfs.yml +# endif + +ifeq ($(FS_SYNC_STRATEGY),docker-sync) +DOCKER_COMPOSE_FILES := \ +-f docker-compose-sync.yml +endif + +ifndef DOCKER_COMPOSE_FILES +$(error FS_SYNC_STRATEGY is set to $(FS_SYNC_STRATEGY). Must be one of: local-mounts, nfs, docker-sync) +endif + +# All three filesystem synchronization strategy require the main docker-compose.yml file. +DOCKER_COMPOSE_FILES := -f docker-compose.yml $(DOCKER_COMPOSE_FILES) + # Generates a help message. Borrowed from https://github.com/pydanny/cookiecutter-djangopackage. help: ## Display this help message @echo "Please use \`make ' where is one of" @@ -50,15 +79,31 @@ dev.checkout: ## Check out "openedx-release/$OPENEDX_RELEASE" in each repo if se dev.clone: ## Clone service repos to the parent directory ./repo.sh clone -dev.provision.run: ## Provision all services with local mounted directories - DOCKER_COMPOSE_FILES="-f docker-compose.yml -f docker-compose-host.yml" ./provision.sh +# dev.provision.run: ## Provision all services with local mounted directories +# DOCKER_COMPOSE_FILES="-f docker-compose.yml -f docker-compose-host.yml" ./provision.sh + +dev.echo: + @echo "Testing ... $(OPENEDX_RELEASE) $(COMPOSE_PROJECT_NAME) $(FS_SYNC_STRATEGY)" + +dev.provision.services: ## Provision default services with local mounted directories + # We provision all default services as well as 'e2e' (end-to-end tests). + # e2e is not part of `DEFAULT_SERVICES` because it isn't a service; + # it's just a way to tell ./provision.sh that the fake data for end-to-end + # tests should be prepared. + $(WINPTY) DOCKER_COMPOSE_FILES="$(DOCKER_COMPOSE_FILES)" bash ./provision.sh $(DEFAULT_SERVICES)+e2e -dev.provision: | check-memory dev.clone dev.provision.run stop ## Provision dev environment with all services stopped +dev.provision.services.%: ## Provision specified services with local mounted directories, separated by plus signs + $(WINPTY) DOCKER_COMPOSE_FILES="$(DOCKER_COMPOSE_FILES)" bash ./provision.sh $* + +dev.provision: | check-memory dev.clone dev.provision.services stop ## Provision dev environment with all services stopped + +dev.cache-programs: ## Copy programs from Discovery to Memcached for use in LMS. + $(WINPTY) bash ./programs/provision.sh cache dev.provision.xqueue: | check-memory dev.provision.xqueue.run stop stop.xqueue dev.provision.xqueue.run: - DOCKER_COMPOSE_FILES="-f docker-compose.yml -f docker-compose-xqueue.yml" ./provision-xqueue.sh + DOCKER_COMPOSE_FILES=$(DOCKER_COMPOSE_FILES) ./provision-xqueue.sh dev.reset: | down dev.repo.reset pull dev.up static update-db ## Attempts to reset the local devstack to a the master working state @@ -68,16 +113,33 @@ dev.status: ## Prints the status of all git repositories dev.repo.reset: ## Attempts to reset the local repo checkouts to the master working state ./repo.sh reset -dev.up: | check-memory ## Bring up all services with host volumes - bash -c 'docker-compose -f docker-compose.yml -f docker-compose-host.yml -f docker-compose-themes.yml up -d' - @# Comment out this next line if you want to save some time and don't care about catalog programs - ./programs/provision.sh cache >/dev/null +# dev.up: | check-memory ## Bring up all services with host volumes +# bash -c 'docker-compose -f docker-compose.yml -f docker-compose-host.yml -f docker-compose-themes.yml up -d' +# @# Comment out this next line if you want to save some time and don't care about catalog programs +# ./programs/provision.sh cache >/dev/null + +dev.up: dev.up.$(DEFAULT_SERVICES) check-memory ## Bring up default services. + +dev.up.%: | check-memory ## Bring up specific services (separated by plus-signs) and their dependencies with host volumes. + docker-compose $(DOCKER_COMPOSE_FILES) up -d $$(echo $* | tr + " ") +ifeq ($(ALWAYS_CACHE_PROGRAMS),true) + make dev.cache-programs +endif + +dev.up.without-deps.%: ## Bring up specific services (separated by plus-signs) without dependencies. + docker-compose $(DOCKER_COMPOSE_FILES) up --d --no-deps $$(echo $* | tr + " ") + +dev.up.with-programs: dev.up dev.cache-programs ## Bring up a all services and cache programs in LMS. + +dev.up.with-programs.%: dev.up.$* dev.cache-programs ## Bring up a service and its dependencies and cache programs in LMS. + +dev.up.watchers: check-memory dev.up.lms_watcher+studio_watcher ## Bring up asset watcher containers -dev.up.watchers: | check-memory ## Bring up asset watcher containers - docker-compose -f docker-compose-watchers.yml -f docker-compose-themes-watchers.yml up -d +# dev.up.watchers: | check-memory ## Bring up asset watcher containers +# docker-compose -f docker-compose-watchers.yml -f docker-compose-themes-watchers.yml up -d -dev.up.xqueue: | check-memory ## Bring up xqueue, assumes you already have lms running - docker-compose -f docker-compose.yml -f docker-compose-xqueue.yml -f docker-compose-host.yml up -d +# dev.up.xqueue: | check-memory ## Bring up xqueue, assumes you already have lms running +# docker-compose -f docker-compose.yml -f docker-compose-xqueue.yml -f docker-compose-host.yml up -d dev.up.all: | dev.up dev.up.watchers ## Bring up all services with host volumes, including watchers @@ -187,7 +249,7 @@ lms-watcher-shell: ## Run a shell on the LMS watcher container docker attach "$$(make --silent dev.print-container.$*)" lms-restart: ## Kill the LMS Django development server. The watcher process will restart it. - docker-compose exec lms bash -c 'kill $$(ps aux | grep "manage.py lms" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'kill $$(ps aux | grep "manage.py lms" | egrep -v "while|grep" | awk "{print \$$2}")' studio-shell: ## Run a shell on the Studio container docker-compose $(DOCKER_COMPOSE_FILES) exec studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open @@ -196,13 +258,13 @@ studio-watcher-shell: ## Run a shell on the studio watcher container docker-compose $(DOCKER_COMPOSE_FILES) exec studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-restart: ## Kill the LMS Django development server. The watcher process will restart it. - docker-compose exec studio bash -c 'kill $$(ps aux | grep "manage.py cms" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'kill $$(ps aux | grep "manage.py cms" | egrep -v "while|grep" | awk "{print \$$2}")' xqueue-shell: ## Run a shell on the XQueue container docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue-restart: ## Kill the XQueue development server. The watcher process will restart it. - docker-compose exec xqueue bash -c 'kill $$(ps aux | grep "manage.py runserver" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue bash -c 'kill $$(ps aux | grep "manage.py runserver" | egrep -v "while|grep" | awk "{print \$$2}")' %-restart-devserver: ## Kill a service's Django development server. The watcher process should restart it. docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' @@ -265,7 +327,7 @@ dev.provision.analytics_pipeline.run: DOCKER_COMPOSE_FILES="-f docker-compose.yml -f docker-compose-host.yml -f docker-compose-analytics-pipeline.yml" ./provision-analytics-pipeline.sh analytics-pipeline-shell: ## Run a shell on the analytics pipeline container - docker-compose exec analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open dev.up.analytics_pipeline: | check-memory ## Bring up analytics pipeline services docker-compose -f docker-compose.yml -f docker-compose-analytics-pipeline.yml -f docker-compose-host.yml up -d analyticspipeline diff --git a/options.mk b/options.mk index b09035515b..7e5e25aac8 100644 --- a/options.mk +++ b/options.mk @@ -16,9 +16,9 @@ DEVSTACK_WORKSPACE ?= $(shell pwd)/.. # otherwise, it defaults to `devstack`. # Be sure to bring down services before changing the value of `COMPOSE_PROJECT_NAME`. ifdef OPENEDX_RELEASE - COMPOSE_PROJECT_NAME ?= devstack-${OPENEDX_RELEASE} + COMPOSE_PROJECT_NAME := devstack-${OPENEDX_RELEASE} else - COMPOSE_PROJECT_NAME ?= devstack + COMPOSE_PROJECT_NAME := devstack endif # Name of Docker Compose project. @@ -47,7 +47,7 @@ FS_SYNC_STRATEGY ?= local-mounts # TODO: Re-evaluate this list and consider paring it down to a tighter core. # The current value was chosen such that it would not change the existing # Devstack behavior. -DEFAULT_SERVICES ?= lms+studio+ecommerce+discovery+xqueue+credentials+forum+edx_notes_api+gradebook+frontend-app-publisher +DEFAULT_SERVICES ?= lms+studio+ecommerce+discovery+xqueue+credentials+forum+edx_notes_api # List of all services with database migrations. # Services must provide a Makefile target named: $(service)-update-db