diff --git a/Makefile b/Makefile index 2153c0d246..3d7af06317 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,46 @@ COMPOSE_PROJECT_NAME=devstack export DEVSTACK_WORKSPACE export COMPOSE_PROJECT_NAME -include *.mk +# include *.mk + +# Include options (configurable through options.local.mk) +include options.mk + +# Include local overrides to options. +# You can use this file to configure your Devstack. It is ignored by git. +-include options.local.mk # Prefix with hyphen to tolerate absence of file. + +# Include local makefile with additional targets. +-include local.mk # Prefix with hyphen to tolerate absence of file. + +ifeq ($(FS_SYNC_STRATEGY),local-mounts) +DOCKER_COMPOSE_FILES := \ +-f docker-compose-host.yml \ +-f docker-compose-themes.yml \ +-f docker-compose-watchers.yml \ +-f docker-compose-xqueue.yml \ +-f docker-compose-analytics-pipeline.yml \ +-f docker-compose-marketing-site.yml +endif + +# ifeq ($(FS_SYNC_STRATEGY),nfs) +# DOCKER_COMPOSE_FILES := \ +# -f docker-compose-host-nfs.yml \ +# -f docker-compose-themes-nfs.yml \ +# -f docker-compose-watchers-nfs.yml +# endif + +ifeq ($(FS_SYNC_STRATEGY),docker-sync) +DOCKER_COMPOSE_FILES := \ +-f docker-compose-sync.yml +endif + +ifndef DOCKER_COMPOSE_FILES +$(error FS_SYNC_STRATEGY is set to $(FS_SYNC_STRATEGY). Must be one of: local-mounts, nfs, docker-sync) +endif + +# All three filesystem synchronization strategy require the main docker-compose.yml file. +DOCKER_COMPOSE_FILES := -f docker-compose.yml $(DOCKER_COMPOSE_FILES) # Generates a help message. Borrowed from https://github.com/pydanny/cookiecutter-djangopackage. help: ## Display this help message @@ -40,15 +79,31 @@ dev.checkout: ## Check out "openedx-release/$OPENEDX_RELEASE" in each repo if se dev.clone: ## Clone service repos to the parent directory ./repo.sh clone -dev.provision.run: ## Provision all services with local mounted directories - DOCKER_COMPOSE_FILES="-f docker-compose.yml -f docker-compose-host.yml" ./provision.sh +# dev.provision.run: ## Provision all services with local mounted directories +# DOCKER_COMPOSE_FILES="-f docker-compose.yml -f docker-compose-host.yml" ./provision.sh -dev.provision: | check-memory dev.clone dev.provision.run stop ## Provision dev environment with all services stopped +dev.echo: + @echo "Testing ... $(OPENEDX_RELEASE) $(COMPOSE_PROJECT_NAME) $(FS_SYNC_STRATEGY)" + +dev.provision.services: ## Provision default services with local mounted directories + # We provision all default services as well as 'e2e' (end-to-end tests). + # e2e is not part of `DEFAULT_SERVICES` because it isn't a service; + # it's just a way to tell ./provision.sh that the fake data for end-to-end + # tests should be prepared. + $(WINPTY) DOCKER_COMPOSE_FILES="$(DOCKER_COMPOSE_FILES)" bash ./provision.sh $(DEFAULT_SERVICES)+e2e + +dev.provision.services.%: ## Provision specified services with local mounted directories, separated by plus signs + $(WINPTY) DOCKER_COMPOSE_FILES="$(DOCKER_COMPOSE_FILES)" bash ./provision.sh $* + +dev.provision: | check-memory dev.clone dev.provision.services stop ## Provision dev environment with all services stopped + +dev.cache-programs: ## Copy programs from Discovery to Memcached for use in LMS. + $(WINPTY) bash ./programs/provision.sh cache dev.provision.xqueue: | check-memory dev.provision.xqueue.run stop stop.xqueue dev.provision.xqueue.run: - DOCKER_COMPOSE_FILES="-f docker-compose.yml -f docker-compose-xqueue.yml" ./provision-xqueue.sh + DOCKER_COMPOSE_FILES=$(DOCKER_COMPOSE_FILES) ./provision-xqueue.sh dev.reset: | down dev.repo.reset pull dev.up static update-db ## Attempts to reset the local devstack to a the master working state @@ -58,19 +113,40 @@ dev.status: ## Prints the status of all git repositories dev.repo.reset: ## Attempts to reset the local repo checkouts to the master working state ./repo.sh reset -dev.up: | check-memory ## Bring up all services with host volumes - bash -c 'docker-compose -f docker-compose.yml -f docker-compose-host.yml -f docker-compose-themes.yml up -d' - @# Comment out this next line if you want to save some time and don't care about catalog programs - ./programs/provision.sh cache >/dev/null +# dev.up: | check-memory ## Bring up all services with host volumes +# bash -c 'docker-compose -f docker-compose.yml -f docker-compose-host.yml -f docker-compose-themes.yml up -d' +# @# Comment out this next line if you want to save some time and don't care about catalog programs +# ./programs/provision.sh cache >/dev/null + +dev.up: dev.up.$(DEFAULT_SERVICES) check-memory ## Bring up default services. + +dev.up.%: | check-memory ## Bring up specific services (separated by plus-signs) and their dependencies with host volumes. + docker-compose $(DOCKER_COMPOSE_FILES) up -d $$(echo $* | tr + " ") +ifeq ($(ALWAYS_CACHE_PROGRAMS),true) + make dev.cache-programs +endif + +dev.up.without-deps.%: ## Bring up specific services (separated by plus-signs) without dependencies. + docker-compose $(DOCKER_COMPOSE_FILES) up --d --no-deps $$(echo $* | tr + " ") -dev.up.watchers: | check-memory ## Bring up asset watcher containers - docker-compose -f docker-compose-watchers.yml -f docker-compose-themes-watchers.yml up -d +dev.up.with-programs: dev.up dev.cache-programs ## Bring up a all services and cache programs in LMS. -dev.up.xqueue: | check-memory ## Bring up xqueue, assumes you already have lms running - docker-compose -f docker-compose.yml -f docker-compose-xqueue.yml -f docker-compose-host.yml up -d +dev.up.with-programs.%: dev.up.$* dev.cache-programs ## Bring up a service and its dependencies and cache programs in LMS. + +dev.up.watchers: check-memory dev.up.lms_watcher+studio_watcher ## Bring up asset watcher containers + +# dev.up.watchers: | check-memory ## Bring up asset watcher containers +# docker-compose -f docker-compose-watchers.yml -f docker-compose-themes-watchers.yml up -d + +# dev.up.xqueue: | check-memory ## Bring up xqueue, assumes you already have lms running +# docker-compose -f docker-compose.yml -f docker-compose-xqueue.yml -f docker-compose-host.yml up -d dev.up.all: | dev.up dev.up.watchers ## Bring up all services with host volumes, including watchers +dev.print-container.%: ## Get the ID of the running container for a given service. + @# Can be run as ``make --silent dev.print-container.$service`` for just ID. + @echo $$(docker-compose $(DOCKER_COMPOSE_FILES) ps --quiet $*) + dev.sync.daemon.start: ## Start the docker-sycn daemon docker-sync start @@ -87,7 +163,7 @@ provision: | dev.provision ## This command will be deprecated in a future releas stop: ## Stop all services (test -d .docker-sync && docker-sync stop) || true ## Ignore failure here - docker-compose stop + docker-compose $(DOCKER_COMPOSE_FILES) stop stop.watchers: ## Stop asset watchers docker-compose -f docker-compose-watchers.yml stop @@ -123,87 +199,96 @@ pull.xqueue: ## Update XQueue Docker images docker-compose -f docker-compose-xqueue.yml pull --parallel validate: ## Validate the devstack configuration - docker-compose config + docker-compose $(DOCKER_COMPOSE_FILES) config backup: ## Write all data volumes to the host. - docker run --rm --volumes-from edx.devstack.mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql - docker run --rm --volumes-from edx.devstack.mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db - docker run --rm --volumes-from edx.devstack.elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data + docker run --rm --volumes-from -T $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql + docker run --rm --volumes-from -T $$(make -s dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db + docker run --rm --volumes-from -T $$(make -s dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data restore: ## Restore all data volumes from the host. WARNING: THIS WILL OVERWRITE ALL EXISTING DATA! - docker run --rm --volumes-from edx.devstack.mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz - docker run --rm --volumes-from edx.devstack.mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz - docker run --rm --volumes-from edx.devstack.elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz + docker run --rm --volumes-from -T $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz + docker run --rm --volumes-from -T $$(make -s dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz + docker run --rm --volumes-from -T $$(make -s dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz # TODO: Print out help for this target. Even better if we can iterate over the # services in docker-compose.yml, and print the actual service names. %-shell: ## Run a shell on the specified service container - docker exec -it edx.devstack.$* /bin/bash + docker-compose $(DOCKER_COMPOSE_FILES) exec $* /bin/bash -credentials-shell: - docker exec -it edx.devstack.credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' +credentials-shell: ## Run a shell on the credentials container + docker-compose $(DOCKER_COMPOSE_FILES) exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' discovery-shell: ## Run a shell on the discovery container - docker exec -it edx.devstack.discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open ecommerce-shell: ## Run a shell on the ecommerce container - docker exec -it edx.devstack.ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open e2e-shell: ## Start the end-to-end tests container with a shell - docker run -it --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash + docker-compose run --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash %-update-db: ## Run migrations for the specified service container - docker exec -t edx.devstack.$* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' + docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' studio-update-db: ## Run migrations for the Studio container - docker exec -t edx.devstack.studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' + docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' lms-update-db: ## Run migrations LMS container - docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' + docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' update-db: | studio-update-db lms-update-db discovery-update-db ecommerce-update-db credentials-update-db ## Run the migrations for all services lms-shell: ## Run a shell on the LMS container - docker exec -it edx.devstack.lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open lms-watcher-shell: ## Run a shell on the LMS watcher container - docker exec -it edx.devstack.lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open %-attach: ## Attach to the specified service container process to use the debugger & see logs. - docker attach edx.devstack.$* + docker attach "$$(make --silent dev.print-container.$*)" lms-restart: ## Kill the LMS Django development server. The watcher process will restart it. - docker exec -t edx.devstack.lms bash -c 'kill $$(ps aux | grep "manage.py lms" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'kill $$(ps aux | grep "manage.py lms" | egrep -v "while|grep" | awk "{print \$$2}")' studio-shell: ## Run a shell on the Studio container - docker exec -it edx.devstack.studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-watcher-shell: ## Run a shell on the studio watcher container - docker exec -it edx.devstack.studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open studio-restart: ## Kill the LMS Django development server. The watcher process will restart it. - docker exec -t edx.devstack.studio bash -c 'kill $$(ps aux | grep "manage.py cms" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'kill $$(ps aux | grep "manage.py cms" | egrep -v "while|grep" | awk "{print \$$2}")' xqueue-shell: ## Run a shell on the XQueue container - docker exec -it edx.devstack.xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue-restart: ## Kill the XQueue development server. The watcher process will restart it. - docker exec -t edx.devstack.xqueue bash -c 'kill $$(ps aux | grep "manage.py runserver" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue bash -c 'kill $$(ps aux | grep "manage.py runserver" | egrep -v "while|grep" | awk "{print \$$2}")' + +%-restart-devserver: ## Kill a service's Django development server. The watcher process should restart it. + docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' xqueue_consumer-shell: ## Run a shell on the XQueue consumer container - docker exec -it edx.devstack.xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open xqueue_consumer-restart: ## Kill the XQueue development server. The watcher process will restart it. - docker exec -t edx.devstack.xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")' + docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")' %-static: ## Rebuild static assets for the specified service container - docker exec -t edx.devstack.$* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' + docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' + +lms-sass: ## Rebuild sass assets for the LMS container + docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=lms --theme-dirs /edx/app/edx-themes/edx-platform/' lms-static: ## Rebuild static assets for the LMS container - docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/ --themes educateworkforce greermade' + docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/' + +studio-sass: ## Rebuild sass assets for the Studio container + docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=cms --theme-dirs /edx/app/edx-themes/edx-platform/' -studio-static: ## Rebuild static assets for the Studio container (educateworkforce,arm,belizeproject,bmw,caregiver,dlsc,greermade,mahcc,meep,nccte,nwgrc,ptc,scelect,scjru,ts) - docker exec -t edx.devstack.studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/ --themes educateworkforce greermade' +studio-static: ## Rebuild static assets for the Studio container + docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/' static: | credentials-static discovery-static ecommerce-static lms-static studio-static ## Rebuild static assets for all service containers @@ -215,24 +300,24 @@ e2e-tests: ## Run the end-to-end tests against the service containers validate-lms-volume: ## Validate that changes to the local workspace are reflected in the LMS container touch $(DEVSTACK_WORKSPACE)/edx-platform/testfile - docker exec edx.devstack.lms ls /edx/app/edxapp/edx-platform/testfile + docker-compose $(DOCKER_COMPOSE_FILES) exec -T lms ls /edx/app/edxapp/edx-platform/testfile rm $(DEVSTACK_WORKSPACE)/edx-platform/testfile vnc-passwords: ## Get the VNC passwords for the Chrome and Firefox Selenium containers - @docker logs edx.devstack.chrome 2>&1 | grep "VNC password" | tail -1 - @docker logs edx.devstack.firefox 2>&1 | grep "VNC password" | tail -1 + @docker-compose $(DOCKER_COMPOSE_FILES) logs chrome 2>&1 | grep "VNC password" | tail -1 + @docker-compose $(DOCKER_COMPOSE_FILES) logs firefox 2>&1 | grep "VNC password" | tail -1 devpi-password: ## Get the root devpi password for the devpi container - docker-compose exec devpi bash -c "cat /data/server/.serverpassword" + docker-compose $(DOCKER_COMPOSE_FILES) exec devpi bash -c "cat /data/server/.serverpassword" mysql-shell: ## Run a shell on the mysql container - docker-compose exec mysql bash + docker-compose $(DOCKER_COMPOSE_FILES) exec mysql bash mysql-shell-edxapp: ## Run a mysql shell on the edxapp database - docker-compose exec mysql bash -c "mysql edxapp" + docker-compose $(DOCKER_COMPOSE_FILES) exec mysql bash -c "mysql edxapp" mongo-shell: ## Run a shell on the mongo container - docker-compose exec mongo bash + docker-compose $(DOCKER_COMPOSE_FILES) exec mongo bash ### analytics pipeline commands @@ -242,7 +327,7 @@ dev.provision.analytics_pipeline.run: DOCKER_COMPOSE_FILES="-f docker-compose.yml -f docker-compose-host.yml -f docker-compose-analytics-pipeline.yml" ./provision-analytics-pipeline.sh analytics-pipeline-shell: ## Run a shell on the analytics pipeline container - docker exec -it edx.devstack.analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open + docker-compose $(DOCKER_COMPOSE_FILES) exec analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open dev.up.analytics_pipeline: | check-memory ## Bring up analytics pipeline services docker-compose -f docker-compose.yml -f docker-compose-analytics-pipeline.yml -f docker-compose-host.yml up -d analyticspipeline @@ -251,14 +336,14 @@ pull.analytics_pipeline: ## Update analytics pipeline docker images docker-compose -f docker-compose.yml -f docker-compose-analytics-pipeline.yml pull --parallel analytics-pipeline-devstack-test: ## Run analytics pipeline tests in travis build - docker exec -u hadoop -i edx.devstack.analytics_pipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity' + docker-compose $(DOCKER_COMPOSE_FILES) exec -u hadoop -T analytics_pipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity' stop.analytics_pipeline: ## Stop analytics pipeline services docker-compose -f docker-compose.yml -f docker-compose-analytics-pipeline.yml stop docker-compose up -d mysql ## restart mysql as other containers need it hadoop-application-logs-%: ## View hadoop logs by application Id - docker exec -it edx.devstack.analytics_pipeline.nodemanager yarn logs -applicationId $* + docker-compose $(DOCKER_COMPOSE_FILES) exec analytics_pipeline.nodemanager yarn logs -applicationId $* # Provisions studio, ecommerce, and marketing with course(s) in test-course.json # Modify test-course.json before running this make target to generate a custom course diff --git a/Makefile.edx b/Makefile.edx index fd9f8ac83d..fb75ff614e 100644 --- a/Makefile.edx +++ b/Makefile.edx @@ -23,12 +23,12 @@ dev.provision.whitelabel: # The containers must be started with the 'dev.up.e2e_wl_tests' target. # AND the test must be setup using the 'dev.provision.whitelabel' target. whitelabel-tests: - docker run -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e - docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh devstack.whitelabel:/tmp/run_whitelabel_tests.sh - docker exec -t devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh + docker-compose run -T -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e + docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh $(make --silent dev.print-container.devstack.whitelabel)":/tmp/run_whitelabel_tests.sh + docker-compose $(DOCKER_COMPOSE_FILES) exec devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh whitelabel-cleanup: - docker rm --force devstack.whitelabel + docker-compose rm --force devstack.whitelabel whitelabel-shell: ## Start a whitelabel test shell session - docker exec -it devstack.whitelabel env TERM=$(TERM) bash + docker-compose $(DOCKER_COMPOSE_FILES) exec -it devstack.whitelabel env TERM=$(TERM) bash diff --git a/README.rst b/README.rst index 42fdb5fbd3..a9f7149a10 100644 --- a/README.rst +++ b/README.rst @@ -404,6 +404,177 @@ images until you change or unset ``OPENEDX_RELEASE`` again. To work on the master branches and ``latest`` images, unset ``OPENEDX_RELEASE`` or set it to an empty string. +How do I run multiple named Open edX releases on same machine? +-------------------------------------------------------------- +You can have multiple isolated Devstacks provisioned on a single computer now. Follow these directions to switch between the named releases. + +#. Bring down any running containers by issuing a `make stop.all`. +#. The ``COMPOSE_PROJECT_NAME`` variable is used to define Docker namespaced volumes and network based on this value, so changing it will give you a separate set of databases. This is handled for you automatically by setting the ``OPENEDX_RELEASE`` environment variable in ``options.mk`` (e.g. ``COMPOSE_PROJECT_NAME=devstack-juniper.master``. Should you want to manually override this edit the ``options.local.mk`` in the root of this repo and create the file if it does not exist. Change the devstack project name by adding the following line: + ``COMPOSE_PROJECT_NAME=`` (e.g. ``COMPOSE_PROJECT_NAME=secondarydevstack``) +#. Perform steps in `How do I run the images for a named Open edX release?`_ for specific release. +#. Follow the steps in `Getting Started`_ section to update requirements (e.g. ``make requirements``) and provision (e.g. ``make dev.provision``) the new named release containers. + +As a specific example, if ``OPENEDX_RELEASE`` is set in your environment as ``juniper.master``, then ``COMPOSE_PROJECT_NAME`` will default to ``devstack-juniper.master`` instead of ``devstack``. + +The implication of this is that you can switch between isolated Devstack databases by changing the value of the ``OPENEDX_RELEASE`` environment variable. + +Switch between your Devstack releases by doing the following: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +#. Bring down the containers by issuing a ``make stop.all`` for the running release. +#. Follow the instructions from the `How do I run multiple named Open edX releases on same machine?`_ section. +#. Edit the project name in ``options.local.mk`` or set the ``OPENEDX_RELEASE`` environment variable and let the ``COMPOSE_PROJECT_NAME`` be assigned automatically. +#. Bring up the containers with ``make dev.up``. + +**NOTE:** Additional instructions on switching releases using `direnv` can be found in `How do I switch releases using 'direnv'?`_ section. + +Examples of Docker Service Names After Setting the ``COMPOSE_PROJECT_NAME`` variable. Notice that the **devstack-juniper.master** name represents the ``COMPOSE_PROJECT_NAME``. + +- edx.devstack-juniper.master.lms +- edx.devstack-juniper.master.mysql + +Each instance has an isolated set of databases. This could, for example, be used to quickly switch between versions of Open edX without hitting as many issues with migrations, data integrity, etc. + +Unfortunately, this **does not** currently support running Devstacks simultaneously, because we hard-code host port numbers all over the place, and two running containers cannot share the same host port. + +Questions & Troubleshooting – Multiple Named Open edX Releases on Same Machine +------------------------------------------------------------------------------ + +This broke my existing Devstack! +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + See if the troubleshooting of this readme can help resolve your broken devstack first, then try posting on the `Open edX forums `__ to see if you have the same issue as any others. If you think you have found a bug, file a CR ticket. + +I’m getting errors related to ports already being used. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Make sure you bring down your devstack before changing the value of COMPOSE_PROJECT_NAME. If you forgot to, change the COMPOSE_PROJECT_NAME back to its original value, run ``make dev.down``, and then try again. + +I have custom scripts/compose files that integrate with or extend Devstack. Will those still work? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +With the default value of COMPOSE_PROJECT_NAME = devstack, they should still work. If you choose a different COMPOSE_PROJECT_NAME, your extensions will likely break, because the names of containers change along with the project name. + +How do I switch releases using 'direnv'? +---------------------------------------- + +Follow directions in `Switch between your Devstack releases by doing the following:`_ then make the following adjustments. + +Make sure that you have setup each Open edX release in separate directories using `How do I enable environment variables for current directory using 'direnv'?`_ instructions. Open the next release project in a separate code editor, then activate the ``direnv`` environment variables and virtual environment for the next release by using a terminal shell to traverse to the directory with the corresponding release ``.envrc`` file. You may need to issue a ``direnv allow`` command to enable the ``.envrc`` file. + + .. code:: sh + + # You should see something like the following after successfully enabling 'direnv' for the Juniper release. + + direnv: loading ~/open-edx/devstack.juniper/.envrc + direnv: export +DEVSTACK_WORKSPACE +OPENEDX_RELEASE +VIRTUAL_ENV ~PATH + (venv)username@computer-name devstack.juniper % + +**NOTE:** Setting of the ``OPENEDX_RELEASE`` should have been handled within the ``.envrc`` file for named releases only and should not be defined for the ``master`` release. + +How do I enable environment variables for current directory using 'direnv'? +--------------------------------------------------------------------------- +We recommend separating the named releases into different directories, for clarity purposes. You can use `direnv `__ to define different environment variables per directory:: + + .. code:: + + # Example showing directory structure for separate Open edX releases. + + /Users//open-edx – root directory for platform development + |_ ./devstack.master – directory containing all repository information related to the main development release. + |_ ./devstack.juniper – directory containing all repository information related to the Juniper release. + +#. Install `direnv` using instructions on https://direnv.net/. Below you will find additional setup at the time of this writing so refer to latest of `direnv` site for additional configuration needed. + +#. Setup the following configuration to hook `direnv` for local directory environment overrides. There are two examples for BASH or ZSH (Mac OS X) shells. + + .. code:: sh + + ## ~/.bashrc for BASH shell + + ## Hook in `direnv` for local directory environment overrides. + ## https://direnv.net/docs/hook.html + eval "$(direnv hook bash)" + + # https://github.com/direnv/direnv/wiki/Python#bash + show_virtual_env() { + if [[ -n "$VIRTUAL_ENV" && -n "$DIRENV_DIR" ]]; then + echo "($(basename $VIRTUAL_ENV))" + fi + } + export -f show_virtual_env + PS1='$(show_virtual_env)'$PS1 + + # --------------------------------------------------- + + ## ~/.zshrc for ZSH shell for Mac OS X. + + ## Hook in `direnv` for local directory environment setup. + ## https://direnv.net/docs/hook.html + eval "$(direnv hook zsh)" + + # https://github.com/direnv/direnv/wiki/Python#zsh + setopt PROMPT_SUBST + + show_virtual_env() { + if [[ -n "$VIRTUAL_ENV" && -n "$DIRENV_DIR" ]]; then + echo "($(basename $VIRTUAL_ENV))" + fi + } + PS1='$(show_virtual_env)'$PS1 + +#. Setup `layout_python-venv` function to be used in local project directory `.envrc` file. + + .. code:: sh + + ## ~/.config/direnv/direnvrc + + # https://github.com/direnv/direnv/wiki/Python#venv-stdlib-module + + realpath() { + [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}" + } + layout_python-venv() { + local python=${1:-python3} + [[ $# -gt 0 ]] && shift + unset PYTHONHOME + if [[ -n $VIRTUAL_ENV ]]; then + VIRTUAL_ENV=$(realpath "${VIRTUAL_ENV}") + else + local python_version + python_version=$("$python" -c "import platform; print(platform.python_version())") + if [[ -z $python_version ]]; then + log_error "Could not detect Python version" + return 1 + fi + VIRTUAL_ENV=$PWD/.direnv/python-venv-$python_version + fi + export VIRTUAL_ENV + if [[ ! -d $VIRTUAL_ENV ]]; then + log_status "no venv found; creating $VIRTUAL_ENV" + "$python" -m venv "$VIRTUAL_ENV" + fi + + PATH="${VIRTUAL_ENV}/bin:${PATH}" + export PATH + } + +#. Example `.envrc` file used in project directory. Need to make sure that each release root has this unique file. + + .. code:: sh + + # Open edX named release project directory root. + ## /devstack.juniper/.envrc + + # https://discuss.openedx.org/t/docker-devstack-multiple-releases-one-machine/1902/10 + + # This is handled when OPENEDX_RELEASE is set. Leaving this in for manual override. + # export COMPOSE_PROJECT_NAME=devstack-juniper + + export DEVSTACK_WORKSPACE="$(pwd)" + export OPENEDX_RELEASE=juniper.master + export VIRTUAL_ENV="$(pwd)/devstack/venv" + + # https://github.com/direnv/direnv/wiki/Python#virtualenv + layout python-venv + How do I create database dumps? ------------------------------- We use database dumps to speed up provisioning and generally spend less time running migrations. These dumps should be diff --git a/course-generator/create-courses.sh b/course-generator/create-courses.sh index eea96f6589..54e2bc3629 100755 --- a/course-generator/create-courses.sh +++ b/course-generator/create-courses.sh @@ -8,21 +8,21 @@ echo "Parsing options" container_error=false for arg in "$@"; do if [ $arg == "--studio" ]; then - if [ ! "$(docker-compose exec lms bash -c 'echo "Course will be created for studio"; exit $?')" ]; then + if [ ! "$(docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'echo "Course will be created for studio"; exit $?')" ]; then echo "Issue with studio container" container_error=true else studio=true fi elif [ $arg == "--ecommerce" ]; then - if [ ! "$(docker exec -t edx.devstack.ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then + if [ ! "$(docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then echo "Issue with ecommerce container" container_error=true else ecommerce=true fi elif [ $arg == "--marketing" ]; then - if [ ! "$(docker exec -t edx.devstack.marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then + if [ ! "$(docker-compose $DOCKER_COMPOSE_FILES exec marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then echo "Issue with marketing container. Course creation will proceed without marketing container." else marketing=true @@ -49,15 +49,15 @@ done < "${@: -1}" if $studio ; then echo "Creating courses on studio." - docker-compose exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'" + docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'" fi if $ecommerce ; then echo "Creating courses on ecommerce." - docker exec -t edx.devstack.ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" + docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" fi if $marketing ; then echo "Creating courses on marketing." - docker exec -t edx.devstack.marketing bash -c "drush generate_courses '$course_json'" + docker-compose $DOCKER_COMPOSE_FILES exec marketing bash -c "drush generate_courses '$course_json'" fi diff --git a/docker-compose-analytics-pipeline.yml b/docker-compose-analytics-pipeline.yml index eef6b6fa24..8f0a546bc3 100644 --- a/docker-compose-analytics-pipeline.yml +++ b/docker-compose-analytics-pipeline.yml @@ -2,11 +2,15 @@ version: "2.1" services: namenode: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.namenode" image: edxops/analytics_pipeline_hadoop_namenode:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.namenode hostname: namenode environment: - CLUSTER_NAME=devstack + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.namenode ports: - 127.0.0.1:50070:50070 command: ["/run.sh"] @@ -14,13 +18,17 @@ services: - namenode_data:/hadoop/dfs/name datanode: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.datanode" image: edxops/analytics_pipeline_hadoop_datanode:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.datanode hostname: datanode environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" depends_on: - namenode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.datanode ports: - 127.0.0.1:50075:50075 command: ["/run.sh"] @@ -28,8 +36,8 @@ services: - datanode_data:/hadoop/dfs/data resourcemanager: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.resourcemanager" image: edxops/analytics_pipeline_hadoop_resourcemanager:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.resourcemanager hostname: resourcemanager environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" @@ -40,13 +48,17 @@ services: depends_on: - namenode - datanode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.resourcemanager ports: - 127.0.0.1:8088:8088 # resource manager web ui command: ["/run.sh"] nodemanager: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.nodemanager" image: edxops/analytics_pipeline_hadoop_nodemanager:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.nodemanager hostname: nodemanager environment: CORE_CONF_fs_defaultFS: "hdfs://namenode:8020" @@ -60,15 +72,23 @@ services: - resourcemanager - namenode - datanode + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.nodemanager ports: - 127.0.0.1:8042:8042 # node manager web ui - 127.0.0.1:19888:19888 # node manager job history server ui command: ["/run.sh"] sparkmaster: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkmaster" image: edxops/analytics_pipeline_spark_master:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.sparkmaster hostname: sparkmaster + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.sparkmaster ports: - 127.0.0.1:8080:8080 - 127.0.0.1:7077:7077 # spark master port @@ -76,25 +96,34 @@ services: - 127.0.0.1:18080:18080 # spark history server sparkworker: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkworker" image: edxops/analytics_pipeline_spark_worker:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline.sparkworker hostname: sparkworker depends_on: - sparkmaster environment: - SPARK_MASTER=spark://sparkmaster:7077 + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.sparkworker ports: - 127.0.0.1:8081:8081 # spark worker UI vertica: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.vertica" image: sumitchawla/vertica:latest - container_name: edx.devstack.analytics_pipeline.vertica + hostname: vertica + networks: + default: + aliases: + - edx.devstack.analytics_pipeline.vertica volumes: - vertica_data:/home/dbadmin/docker analyticspipeline: + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline" image: edxops/analytics_pipeline:${OPENEDX_RELEASE:-latest} - container_name: edx.devstack.analytics_pipeline hostname: analyticspipeline volumes: - ${DEVSTACK_WORKSPACE}/edx-analytics-pipeline:/edx/app/analytics_pipeline/analytics_pipeline @@ -108,6 +137,10 @@ services: - sparkworker - elasticsearch - vertica + networks: + default: + aliases: + - edx.devstack.analytics_pipeline ports: - 127.0.0.1:4040:4040 # spark web UI environment: diff --git a/docker-compose-marketing-site.yml b/docker-compose-marketing-site.yml index 4bd2693b7a..bc48b89ce1 100644 --- a/docker-compose-marketing-site.yml +++ b/docker-compose-marketing-site.yml @@ -7,7 +7,7 @@ services: - MARKETING_SITE_ROOT="http://localhost:8080" marketing: - container_name: edx.devstack.marketing + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.marketing" depends_on: - mysql - memcached @@ -26,5 +26,9 @@ services: # IP address of your machine to enable debugging (IP_ADDRESS set in .env file) - XDEBUG_CONFIG=remote_host=${XDEBUG_IP_ADDRESS:-127.0.0.1} image: edxops/edx-mktg:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.marketing ports: - "8080:80" diff --git a/docker-compose-watchers.yml b/docker-compose-watchers.yml index 0f55930d19..f464815f43 100644 --- a/docker-compose-watchers.yml +++ b/docker-compose-watchers.yml @@ -3,11 +3,15 @@ version: "2.1" services: lms_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT --theme-dirs=/edx/app/edx-themes/edx-platform/ --themes=educateworkforce,greermade; sleep 2; done' - container_name: edx.devstack.lms_watcher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.lms_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.lms_watcher volumes: - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform:cached - edxapp_lms_assets:/edx/var/edxapp/staticfiles/ @@ -17,11 +21,15 @@ services: studio_watcher: command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT --theme-dirs=/edx/app/edx-themes/edx-platform/ --themes=educateworkforce,greermade; sleep 2; done' - container_name: edx.devstack.studio_watcher + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio_watcher" environment: BOK_CHOY_HOSTNAME: edx.devstack.studio_watcher ASSET_WATCHER_TIMEOUT: 12 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.studio_watcher volumes: - edxapp_studio_assets:/edx/var/edxapp/staticfiles/ - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform:cached diff --git a/docker-compose-xqueue.yml b/docker-compose-xqueue.yml index bf4a73ac54..89e987c5ba 100644 --- a/docker-compose-xqueue.yml +++ b/docker-compose-xqueue.yml @@ -2,19 +2,27 @@ version: "2.1" services: xqueue: - container_name: edx.devstack.xqueue + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue" image: edxops/xqueue:${OPENEDX_RELEASE:-latest} command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py runserver 0.0.0.0:18040 ; sleep 2; done' volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file + networks: + default: + aliases: + - edx.devstack.xqueue ports: - 18040:18040 xqueue_consumer: - container_name: edx.devstack.xqueue_consumer + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue_consumer" image: edxops/xqueue:${OPENEDX_RELEASE:-latest} command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py run_consumer ; sleep 2; done' + networks: + default: + aliases: + - edx.devstack.xqueue_consumer volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file diff --git a/docker-compose.yml b/docker-compose.yml index 027bac2e94..9814a3609e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,9 +13,14 @@ version: "2.1" services: # Third-party services chrome: - container_name: edx.devstack.chrome + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.chrome" + hostname: chrome.devstack.edx image: edxops/chrome:${OPENEDX_RELEASE:-latest} shm_size: 2g + networks: + default: + aliases: + - edx.devstack.chrome ports: - "15900:5900" volumes: # for file uploads @@ -23,8 +28,13 @@ services: - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data elasticsearch: - container_name: edx.devstack.elasticsearch + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.elasticsearch" + hostname: elasticsearch.devstack.edx image: edxops/elasticsearch:devstack + networks: + default: + aliases: + - edx.devstack.elasticsearch # TODO: What to do about these forwarded ports? They'll conflict with ports forwarded by the Vagrant VM. # ports: # - "9200:9200" @@ -34,9 +44,14 @@ services: - elasticsearch_data:/usr/share/elasticsearch/logs firefox: - container_name: edx.devstack.firefox + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.firefox" + hostname: firefox.devstack.edx image: edxops/firefox:${OPENEDX_RELEASE:-latest} shm_size: 2g + networks: + default: + aliases: + - edx.devstack.firefox ports: - "25900:5900" volumes: # for file uploads @@ -44,8 +59,13 @@ services: - ../edx-platform/common/test/data:/edx/app/edxapp/edx-platform/common/test/data memcached: - container_name: edx.devstack.memcached + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.memcached" + hostname: memcached.devstack.edx image: memcached:1.4.24 + networks: + default: + aliases: + - edx.devstack.memcached # ports: # - "11211:11211" @@ -54,8 +74,13 @@ services: # to conserve disk space, and disable the journal for a minor performance gain. # See https://docs.mongodb.com/v3.0/reference/program/mongod/#options for complete details. command: mongod --smallfiles --nojournal --storageEngine wiredTiger - container_name: edx.devstack.mongo + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mongo" + hostname: mongo.devstack.edx image: mongo:3.2.16 + networks: + default: + aliases: + - edx.devstack.mongo # ports: # - "27017:27017" volumes: @@ -63,20 +88,27 @@ services: mysql: command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci - container_name: edx.devstack.mysql + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql" + hostname: mysql.devstack.edx environment: MYSQL_ROOT_PASSWORD: "" MYSQL_ALLOW_EMPTY_PASSWORD: "yes" image: mysql:5.6 + networks: + default: + aliases: + - edx.devstack.mysql ports: - "33060:3306" volumes: - mysql_data:/var/lib/mysql + # ================================================ # edX services credentials: command: bash -c 'source /edx/app/credentials/credentials_env && while true; do python /edx/app/credentials/credentials/manage.py runserver 0.0.0.0:18150; sleep 2; done' - container_name: edx.devstack.credentials + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.credentials" + hostname: credentials.devstack.edx depends_on: - mysql - memcached @@ -89,12 +121,17 @@ services: SOCIAL_AUTH_EDX_OIDC_URL_ROOT: http://edx.devstack.lms:18000/oauth2 ENABLE_DJANGO_TOOLBAR: 1 image: edxops/credentials:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.credentials ports: - "18150:18150" discovery: command: bash -c 'source /edx/app/discovery/discovery_env && while true; do python /edx/app/discovery/discovery/manage.py runserver 0.0.0.0:18381; sleep 2; done' - container_name: edx.devstack.discovery + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.discovery" + hostname: discovery.devstack.edx depends_on: - mysql - elasticsearch @@ -106,6 +143,10 @@ services: TEST_ELASTICSEARCH_URL: "http://edx.devstack.elasticsearch:9200" ENABLE_DJANGO_TOOLBAR: 1 image: edxops/discovery:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.discovery ports: - "18381:18381" volumes: @@ -113,7 +154,8 @@ services: ecommerce: command: bash -c 'source /edx/app/ecommerce/ecommerce_env && while true; do python /edx/app/ecommerce/ecommerce/manage.py runserver 0.0.0.0:18130; sleep 2; done' - container_name: edx.devstack.ecommerce + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.ecommerce" + hostname: ecommerce.devstack.edx depends_on: - mysql - memcached @@ -123,12 +165,33 @@ services: environment: ENABLE_DJANGO_TOOLBAR: 0 image: edxops/ecommerce:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.ecommerce ports: - "18130:18130" + forum: + command: bash -c 'source /edx/app/forum/ruby_env && source /edx/app/forum/devstack_forum_env && cd /edx/app/forum/cs_comments_service && bundle install && while true; do ruby app.rb -o 0.0.0.0 ; sleep 2; done' + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.forum" + hostname: forum.devstack.edx + depends_on: + - mongo + - memcached + - elasticsearch + image: edxops/forum:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.forum + ports: + - "44567:4567" + lms: command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack_docker; sleep 2; done' - container_name: edx.devstack.lms + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms" + hostname: lms.devstack.edx depends_on: - devpi - mysql @@ -144,6 +207,10 @@ services: EDXAPP_TEST_MONGO_HOST: edx.devstack.mongo NO_PYTHON_UNINSTALL: 1 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.lms ports: - "18000:18000" - "19876:19876" # JS test debugging @@ -154,12 +221,17 @@ services: edx_notes_api: command: bash -c 'source /edx/app/edx_notes_api/edx_notes_api_env && while true; do python /edx/app/edx_notes_api/edx_notes_api/manage.py runserver 0.0.0.0:18120 --settings notesserver.settings.devstack; sleep 2; done' - container_name: edx.devstack.edx_notes_api + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.edx_notes_api" + hostname: edx_notes_api.devstack.edx depends_on: - devpi - elasticsearch - mysql image: edxops/notes:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.edxnotesapi ports: - "18120:18120" environment: @@ -169,12 +241,14 @@ services: DB_PASSWORD: "password" DB_PORT: "3306" DB_USER: "notes001" + DJANGO_WATCHMAN_TIMEOUT: 30 ENABLE_DJANGO_TOOLBAR: 1 ELASTICSEARCH_URL: "http://edx.devstack.elasticsearch:9200" studio: command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py cms runserver 0.0.0.0:18010 --settings devstack_docker; sleep 2; done' - container_name: edx.devstack.studio + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio" + hostname: studio.devstack.edx depends_on: - devpi - mysql @@ -190,6 +264,10 @@ services: EDXAPP_TEST_MONGO_HOST: edx.devstack.mongo NO_PYTHON_UNINSTALL: 1 image: edxops/edxapp:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.studio ports: - "18010:18010" - "19877:19877" # JS test debugging @@ -198,20 +276,14 @@ services: volumes: - edxapp_studio_assets:/edx/var/edxapp/staticfiles/ - forum: - command: bash -c 'source /edx/app/forum/ruby_env && source /edx/app/forum/devstack_forum_env && cd /edx/app/forum/cs_comments_service && bundle install && while true; do ruby app.rb -o 0.0.0.0 ; sleep 2; done' - container_name: edx.devstack.forum - depends_on: - - mongo - - memcached - - elasticsearch - image: edxops/forum:${OPENEDX_RELEASE:-latest} - ports: - - "44567:4567" - devpi: - container_name: edx.devstack.devpi + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.devpi" + hostname: devpi.devstack.edx image: edxops/devpi:${OPENEDX_RELEASE:-latest} + networks: + default: + aliases: + - edx.devstack.devpi ports: - "3141:3141" volumes: diff --git a/dump-db.sh b/dump-db.sh index 9d9c135704..50e22948fb 100755 --- a/dump-db.sh +++ b/dump-db.sh @@ -14,5 +14,5 @@ then fi echo "Dumping the $1 database..." -docker exec -i edx.devstack.mysql mysqldump --add-drop-database --skip-add-drop-table -B $1 > $1.sql +docker-compose exec mysql base -c "mysqldump --add-drop-database --skip-add-drop-table -B $1" > $1.sql echo "Finished dumping the $1 database!" diff --git a/gather-feature-toggle-state.sh b/gather-feature-toggle-state.sh new file mode 100644 index 0000000000..2ed9946195 --- /dev/null +++ b/gather-feature-toggle-state.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# gather-feature-toggle-state.sh + +# as part of our feature toggle reporting utility, this script can be run +# against a running devstack to collect the state of all of feature toggles +# (django-waffle, waffle-utils) currently configured for each of the IDAs +# that use them. + +if [ -e feature-toggle-data ]; then + rm -Rf feature-toggle-data +fi +mkdir feature-toggle-data + +docker exec -t edx.devstack.credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py dumpdata waffle --format=json > /edx/app/credentials/credentials/credentials_waffle.json' +docker cp edx.devstack.credentials:/edx/app/credentials/credentials/credentials_waffle.json feature-toggle-data +docker exec -t edx.devstack.credentials bash -c 'rm /edx/app/credentials/credentials/credentials_waffle.json' + +docker exec -t edx.devstack.discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py dumpdata waffle --format=json > /edx/app/discovery/discovery/discovery_waffle.json' +docker cp edx.devstack.discovery:/edx/app/discovery/discovery/discovery_waffle.json feature-toggle-data +docker exec -t edx.devstack.discovery bash -c '> /edx/app/discovery/discovery/discovery_waffle.json' + +docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py dumpdata waffle --format=json > /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' +docker cp edx.devstack.ecommerce:/edx/app/ecommerce/ecommerce/ecommerce_waffle.json feature-toggle-data +docker exec -t edx.devstack.ecommerce bash -c 'rm /edx/app/ecommerce/ecommerce/ecommerce_waffle.json' + +docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle --format=json > /edx/app/edxapp/edx-platform/lms_waffle.json' +docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms dumpdata waffle_utils --format=json > /edx/app/edxapp/edx-platform/lms_waffle_utils.json' +docker cp edx.devstack.lms:/edx/app/edxapp/edx-platform/lms_waffle.json feature-toggle-data +docker cp edx.devstack.lms:/edx/app/edxapp/edx-platform/lms_waffle_utils.json feature-toggle-data +docker exec -t edx.devstack.lms bash -c 'rm /edx/app/edxapp/edx-platform/lms_waffle*.json' diff --git a/load-db.sh b/load-db.sh index 24b70fada5..9b9aaa6c75 100755 --- a/load-db.sh +++ b/load-db.sh @@ -17,5 +17,6 @@ then fi echo "Loading the $1 database..." -docker exec -i edx.devstack.mysql mysql -uroot $1 < $1.sql +mysql_container=$(make -s dev.print-container.mysql) +docker exec "$mysql_container" bash -c "mysql -uroot $1" < $1.sql echo "Finished loading the $1 database!" diff --git a/marketing.mk b/marketing.mk index 6932e36395..349872d450 100644 --- a/marketing.mk +++ b/marketing.mk @@ -3,7 +3,7 @@ help-marketing: ## Display this help message @perl -nle'print $& if m{^[\.a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | grep marketing | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m %-25s\033[0m %s\n", $$1, $$2}' marketing-shell: ## Run a shell on the marketing site container - docker exec -it edx.devstack.marketing env TERM=$(TERM) bash + docker-compose $(DOCKER_COMPOSE_FILES) exec marketing env TERM=$(TERM) bash -c 'cd /edx/app/edx-mktg/edx-mktg; exec /bin/bash -sh' stop-marketing: ## Stop all services (including the marketing site) with host volumes docker-compose -f docker-compose.yml -f docker-compose-host.yml -f docker-compose-marketing-site.yml -f docker-compose-marketing-site-host.yml stop diff --git a/options.mk b/options.mk new file mode 100644 index 0000000000..7e5e25aac8 --- /dev/null +++ b/options.mk @@ -0,0 +1,63 @@ +# DEFAULT DEVSTACK OPTIONS +# Included into Makefile and exported to command environment. +# Defaults are listed in this file. +# Local git-ignored overrides can be configured by creating `options.local.mk`. +# Variables are set here with ?= to allow for overriding them on the command line. + +# Folder in which we looks for repositories. +# Defaults to parent. +DEVSTACK_WORKSPACE ?= $(shell pwd)/.. + +# Name of Docker Compose project. +# Volumes and network are namespaced based on this value, +# so changing it will give you a separate set of databases. +# See https://docs.docker.com/compose/reference/envvars/#compose_project_name +# If OPENEDX_RELAESE is defined, defaults to `devstack-${OPENEDX_RELEASE}`; +# otherwise, it defaults to `devstack`. +# Be sure to bring down services before changing the value of `COMPOSE_PROJECT_NAME`. +ifdef OPENEDX_RELEASE + COMPOSE_PROJECT_NAME := devstack-${OPENEDX_RELEASE} +else + COMPOSE_PROJECT_NAME := devstack +endif + +# Name of Docker Compose project. +# See https://docs.docker.com/compose/reference/envvars/#compose_project_name +# Defaults to 'devstack'. +COMPOSE_PROJECT_NAME ?= devstack + +# increase Docker Compose HTTP timeout so that devstack provisioning does not fail in unstable networks +COMPOSE_HTTP_TIMEOUT=180 + +# Whether we should always copy programs to LMS cache upon LMS startup. +# If 'true', then run `make dev.cache-programs` whenever we bring up +# containers. +# Defaults to false. +ALWAYS_CACHE_PROGRAMS ?= false + +# FileSystem Synchronization Strategy. +# How should we synchronize files between the host machine and the Docker containers? +# Options are 'local-mount', 'nfs', and 'docker-sync'. +# Note that 'local-mount' is the most tested and supported with edX's Devstack +# and 'docker-sync' the least. +FS_SYNC_STRATEGY ?= local-mounts + +# Services that are pulled, provisioned, run, and checked by default +# when no services are specified manually. +# TODO: Re-evaluate this list and consider paring it down to a tighter core. +# The current value was chosen such that it would not change the existing +# Devstack behavior. +DEFAULT_SERVICES ?= lms+studio+ecommerce+discovery+xqueue+credentials+forum+edx_notes_api + +# List of all services with database migrations. +# Services must provide a Makefile target named: $(service)-update-db +# Note: This list should contain _all_ db-backed services, even if not +# configured to run; the list will be filtered later against $(DEFAULT_SERVICES) +DB_SERVICES ?= credentials discovery ecommerce lms registrar studio + +# List of Makefile targets to run database migrations, in the form $(service)-update-db +# Services will only have their migrations added here +# if the service is present in both $(DEFAULT_SERVICES) and $(DB_SERVICES). +DB_MIGRATION_TARGETS = $(foreach db_service,$(DB_SERVICES),\ + $(if $(filter $(db_service),$(subst +, ,$(DEFAULT_SERVICES))),\ + $(db_service)-update-db)) diff --git a/programs/provision.sh b/programs/provision.sh index 8ce3d1567a..939a719603 100755 --- a/programs/provision.sh +++ b/programs/provision.sh @@ -42,7 +42,7 @@ docker_exec() { /edx/app/$app/$repo/manage.py $cmd " - docker-compose exec "$service" bash -c "$CMDS" + docker-compose $DOCKER_COMPOSE_FILES exec -T "$service" bash -c "$CMDS" } provision_ida() { diff --git a/provision-analytics-pipeline.sh b/provision-analytics-pipeline.sh index a866fe3fe6..1798fc334f 100755 --- a/provision-analytics-pipeline.sh +++ b/provision-analytics-pipeline.sh @@ -21,7 +21,7 @@ docker-compose $DOCKER_COMPOSE_FILES up -d mysql analyticspipeline # Ensure the MySQL server is online and usable echo "Waiting for MySQL" -until docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')" &> /dev/null +until docker-compose exec mysql mysql -uroot -se "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')" &> /dev/null do printf "." sleep 1 @@ -34,24 +34,24 @@ sleep 20 # Analytics pipeline has dependency on lms but we only need its db schema & not full lms. So we'll just load their db # schemas as part of analytics pipeline provisioning. If there is a need of a fully fledged LMS, then provision lms # by following their documentation. -if [[ ! -z "`docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='edxapp'" 2>&1`" ]]; +if [[ ! -z "`docker-compose $DOCKER_COMPOSE_FILES exec mysql mysql -uroot -se "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='edxapp'" 2>&1`" ]]; then echo -e "${GREEN}LMS DB exists, skipping lms schema load.${NC}" else echo -e "${GREEN}LMS DB not found, provisioning lms schema.${NC}" - docker exec -i edx.devstack.mysql mysql -uroot mysql < provision.sql + docker-compose $DOCKER_COMPOSE_FILES exec mysql bash -c 'mysql -uroot mysql' < provision.sql ./load-db.sh edxapp docker-compose $DOCKER_COMPOSE_FILES up -d lms studio - docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' + docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' #Installing prereqs crashes the process docker-compose restart lms # Run edxapp migrations first since they are needed for the service users and OAuth clients - docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' + docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' fi echo -e "${GREEN}LMS database provisioned successfully...${NC}" echo -e "${GREEN}Creating databases and users...${NC}" -docker exec -i edx.devstack.mysql mysql -uroot mysql < provision-analytics-pipeline.sql +docker-compose $DOCKER_COMPOSE_FILES exec mysql bash -c 'mysql -uroot mysql' < provision-analytics-pipeline.sql # initialize hive metastore echo -e "${GREEN}Initializing HIVE metastore...${NC}" @@ -68,8 +68,12 @@ done sleep 10 # for datanode & other services to activate echo -e "${GREEN}Namenode is ready!${NC}" -docker exec -u hadoop -i edx.devstack.analytics_pipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' +<<<<<<< HEAD:provision-analytics-pipeline.sh +docker-compose exec -u hadoop analytics_pipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' docker image prune -f echo -e "${GREEN}Analytics pipeline provisioning complete!${NC}" +======= +docker-compose $DOCKER_COMPOSE_FILES exec -u hadoop analytics_pipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;' +>>>>>>> 357f8a8... Make sure to pass in `DOCKER_COMPOSE_FILES` to `docker-compose exec` commands for services outside of `docker-compose.yml`.:provision-analyticspipeline.sh diff --git a/provision-credentials.sh b/provision-credentials.sh index 32fccdaf61..1029ce339d 100755 --- a/provision-credentials.sh +++ b/provision-credentials.sh @@ -9,20 +9,20 @@ port=18150 docker-compose $DOCKER_COMPOSE_FILES up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements && make production-requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" echo -e "${GREEN}Configuring site for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/ && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker exec -t edx.devstack.${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker-compose $DOCKER_COMPOSE_FILES exec ${name} bash -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-discovery.sh b/provision-discovery.sh index 2ffec8d1d2..d58f79351e 100755 --- a/provision-discovery.sh +++ b/provision-discovery.sh @@ -1,10 +1,10 @@ # Provisioning script for the discovery service ./provision-ida.sh discovery discovery 18381 -docker-compose exec discovery bash -c 'rm -rf /edx/var/discovery/*' -docker-compose exec discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 1 --site-domain localhost:18381 --code edx --name edX --courses-api-url "http://edx.devstack.lms:18000/api/courses/v1/" --ecommerce-api-url "http://edx.devstack.ecommerce:18130/api/v2/" --organizations-api-url "http://edx.devstack.lms:18000/api/organizations/v0/" --oidc-url-root "http://edx.devstack.lms:18000/oauth2" --oidc-key discovery-key --oidc-secret discovery-secret' -docker-compose exec discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py refresh_course_metadata' -docker-compose exec discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py update_index --disable-change-limit' +docker-compose $DOCKER_COMPOSE_FILES exec -T discovery bash -c 'rm -rf /edx/var/discovery/*' +docker-compose $DOCKER_COMPOSE_FILES exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 1 --site-domain localhost:18381 --code edx --name edX --courses-api-url "http://edx.devstack.lms:18000/api/courses/v1/" --ecommerce-api-url "http://edx.devstack.ecommerce:18130/api/v2/" --organizations-api-url "http://edx.devstack.lms:18000/api/organizations/v0/" --lms-url "http://edx.devstack.lms:18000/"' +docker-compose $DOCKER_COMPOSE_FILES exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py refresh_course_metadata' +docker-compose $DOCKER_COMPOSE_FILES exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py update_index --disable-change-limit' # Add demo program ./programs/provision.sh discovery diff --git a/provision-e2e.sh b/provision-e2e.sh index 47417df310..baee623cd0 100755 --- a/provision-e2e.sh +++ b/provision-e2e.sh @@ -10,14 +10,14 @@ elif [ ! -d "$DEVSTACK_WORKSPACE" ]; then fi # Copy the test course tarball into the studio container -docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz edx.devstack.studio:/tmp/ +docker cp ${DEVSTACK_WORKSPACE}/edx-e2e-tests/upload_files/course.tar.gz "$(make --silent dev.print-container.studio)":/tmp/ # Extract the test course tarball -docker-compose exec studio bash -c 'cd /tmp && tar xzf course.tar.gz' +docker-compose $DOCKER_COMPOSE_FILES exec -T studio bash -c 'cd /tmp && tar xzf course.tar.gz' # Import the course content -docker-compose exec studio bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker import /tmp course' +docker-compose $DOCKER_COMPOSE_FILES exec -T studio bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker import /tmp course' # Clean up the temp files -docker-compose exec studio bash -c 'rm /tmp/course.tar.gz' -docker-compose exec studio bash -c 'rm -r /tmp/course' +docker-compose $DOCKER_COMPOSE_FILES exec -T studio bash -c 'rm /tmp/course.tar.gz' +docker-compose $DOCKER_COMPOSE_FILES exec -T studio bash -c 'rm -r /tmp/course' diff --git a/provision-ecommerce.sh b/provision-ecommerce.sh index b807af9c29..bfefe42217 100755 --- a/provision-ecommerce.sh +++ b/provision-ecommerce.sh @@ -4,6 +4,6 @@ ./provision-ida.sh ecommerce ecommerce 18130 # Configure ecommerce -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --client-id=ecommerce-key --client-secret=ecommerce-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/' -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' -docker exec -t edx.devstack.ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' +docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/' +docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' +docker-compose $DOCKER_COMPOSE_FILES exec ecommerce bash -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' diff --git a/provision-forum.sh b/provision-forum.sh index 793a799b73..8e1f13b864 100755 --- a/provision-forum.sh +++ b/provision-forum.sh @@ -3,4 +3,4 @@ set -o pipefail set -x docker-compose $DOCKER_COMPOSE_FILES up -d forum -docker-compose exec forum bash -c 'source /edx/app/forum/ruby_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/' +docker-compose $DOCKER_COMPOSE_FILES exec -T forum bash -c 'source /edx/app/forum/ruby_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/' diff --git a/provision-ida-user.sh b/provision-ida-user.sh index a18ce1d301..a7fc91572b 100755 --- a/provision-ida-user.sh +++ b/provision-ida-user.sh @@ -5,5 +5,5 @@ client_name=$2 client_port=$3 echo -e "${GREEN}Creating service user and OAuth client for ${app_name}...${NC}" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_oauth2_client "http://localhost:$3" "http://localhost:$3/complete/edx-oidc/" confidential --client_name $2 --client_id "$1-key" --client_secret "$1-secret" --trusted --logout_uri "http://localhost:$3/logout/" --username $1_worker' -- "$app_name" "$client_name" "$client_port" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_oauth2_client "http://localhost:$3" "http://localhost:$3/complete/edx-oidc/" confidential --client_name $2 --client_id "$1-key" --client_secret "$1-secret" --trusted --logout_uri "http://localhost:$3/logout/" --username $1_worker' -- "$app_name" "$client_name" "$client_port" diff --git a/provision-ida.sh b/provision-ida.sh index 02206bd735..34e80ead6a 100755 --- a/provision-ida.sh +++ b/provision-ida.sh @@ -5,17 +5,17 @@ client_port=$3 # The port corresponding to this IDA service in devstack. docker-compose $DOCKER_COMPOSE_FILES up -d $app_name echo -e "${GREEN}Installing requirements for ${app_name}...${NC}" -docker exec -t edx.devstack.${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" echo -e "${GREEN}Running migrations for ${app_name}...${NC}" -docker exec -t edx.devstack.${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" echo -e "${GREEN}Creating super-user for ${app_name}...${NC}" -docker exec -t edx.devstack.${app_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec ${app_name} bash -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" ./provision-ida-user.sh $app_name $client_name $client_port # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${app_name}...${NC}" -docker exec -t edx.devstack.${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" +docker-compose $DOCKER_COMPOSE_FILES exec ${app_name} bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" diff --git a/provision-lms.sh b/provision-lms.sh index 3b7439a06d..991b04e9bb 100755 --- a/provision-lms.sh +++ b/provision-lms.sh @@ -13,33 +13,33 @@ for app in "${apps[@]}"; do docker-compose $DOCKER_COMPOSE_FILES up -d $app done -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' #Installing prereqs crashes the process docker-compose restart lms # Run edxapp migrations first since they are needed for the service users and OAuth clients -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_db --settings devstack_docker' # Create a superuser for edxapp -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user edx edx@example.com --superuser --staff' -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username=\"edx\"); user.set_password(\"edx\"); user.save()" | python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user edx edx@example.com --superuser --staff' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username=\"edx\"); user.set_password(\"edx\"); user.save()" | python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' # Create an enterprise service user for edxapp docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user enterprise_worker enterprise_worker@example.com' # Enable the LMS-E-Commerce integration -docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce' # Create demo course and users -docker-compose exec lms bash -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"' # Fix missing vendor file by clearing the cache -docker-compose exec lms bash -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1' +docker-compose $DOCKER_COMPOSE_FILES exec -T lms bash -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1' # Create static assets for both LMS and Studio for app in "${apps[@]}"; do - docker-compose exec $app bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker' + docker-compose $DOCKER_COMPOSE_FILES exec -T $app bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker' done # Provision a retirement service account user diff --git a/provision-notes.sh b/provision-notes.sh index 4a2c924ab4..9d701ae351 100755 --- a/provision-notes.sh +++ b/provision-notes.sh @@ -5,4 +5,4 @@ # This will build the elasticsearch index for notes. echo -e "${GREEN}Creating indexes for edx_notes_api...${NC}" -docker exec -t edx.devstack.edx_notes_api bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py rebuild_index --noinput' -- edx_notes_api +docker-compose $DOCKER_COMPOSE_FILES exec edx_notes_api bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && python manage.py rebuild_index --noinput' -- edx_notes_api diff --git a/provision-retirement-user.sh b/provision-retirement-user.sh index b3a5dac2e3..9a9f209c6c 100755 --- a/provision-retirement-user.sh +++ b/provision-retirement-user.sh @@ -4,5 +4,5 @@ app_name=$1 user_name=$2 echo -e "${GREEN}Creating retirement service user ${user_name} and DOT Application ${app_name}...${NC}" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" -docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" +docker-compose $DOCKER_COMPOSE_FILES exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" diff --git a/provision-xqueue.sh b/provision-xqueue.sh index 36e55579bb..b1cdc93f59 100755 --- a/provision-xqueue.sh +++ b/provision-xqueue.sh @@ -11,13 +11,13 @@ docker-compose up -d mysql # Ensure the MySQL server is online and usable echo "Waiting for MySQL" -until docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')" &> /dev/null +until docker-compose $DOCKER_COMPOSE_FILES exec -T mysql bash -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 done -docker exec -i edx.devstack.mysql mysql -uroot mysql < provision-xqueue.sql +docker-compose $DOCKER_COMPOSE_FILES exec -T mysql bash -c "mysql -uroot mysql" < provision-xqueue.sql # Run migrations docker-compose $DOCKER_COMPOSE_FILES exec xqueue bash -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py migrate' # Add users that graders use to fetch data, there's one default user in Ansible which is part of our settings diff --git a/provision.sh b/provision.sh index 8324ff18dc..118b30f935 100755 --- a/provision.sh +++ b/provision.sh @@ -21,8 +21,8 @@ NC='\033[0m' # No Color docker-compose up -d mysql mongo # Ensure the MySQL server is online and usable -echo "Waiting for MySQL" -until docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')" &> /dev/null +echo "${GREEN}Waiting for MySQL.${NC}" +until docker-compose $DOCKER_COMPOSE_FILES exec -T mysql bash -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 @@ -32,11 +32,11 @@ done # This can be removed once https://github.com/docker-library/mysql/issues/245 is resolved. sleep 20 -echo -e "MySQL ready" - -echo -e "${GREEN}Creating databases and users...${NC}" -docker exec -i edx.devstack.mysql mysql -uroot mysql < provision.sql -docker exec -i edx.devstack.mongo mongo < mongo-provision.js +# Ensure that the MySQL databases and users are created for all IDAs. +# (A no-op for databases and users that already exist). +echo -e "${GREEN}Ensuring MySQL databases and users exist...${NC}" +docker-compose $DOCKER_COMPOSE_FILES exec -T mysql bash -c "mysql -uroot mysql" < provision.sql +docker-compose $DOCKER_COMPOSE_FILES exec -T mongo mongo < mongo-provision.js ./provision-lms.sh