Skip to content
This repository was archived by the owner on Aug 1, 2024. It is now read-only.

[Backport] Multiple devstack releases with one machine #564

Merged
regisb merged 5 commits into from
Aug 17, 2020
Merged

[Backport] Multiple devstack releases with one machine #564

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
da3a92f
Update `docker` commands to be `docker-compose` to handle unnamed con…
ztraboo Jul 27, 2020
09110f4
Updated `README` to include necessary information when running multip…
ztraboo Jul 28, 2020
357f8a8
Make sure to pass in `DOCKER_COMPOSE_FILES` to `docker-compose exec` …
ztraboo Jul 29, 2020
ffcafdf
Add `xqueue` to default services to provision initially.
ztraboo Aug 10, 2020
35d197b
Make additional adjustments to the documentation for multiple release…
ztraboo Aug 17, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 52 additions & 42 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,10 @@ dev.nfs.%:

dev.up.all: dev.up dev.up.watchers ## Bring up all services with host volumes, including watchers

dev.print-container.%: ## Get the ID of the running container for a given service.
@# Can be run as ``make --silent dev.print-container.$service`` for just ID.
@echo $$(docker-compose $(DOCKER_COMPOSE_FILES) ps --quiet $*)
Comment thread
regisb marked this conversation as resolved.

# TODO: Improve or rip out Docker Sync targets.
# They are not well-fleshed-out and it is not clear if anyone uses them.

Expand All @@ -202,7 +206,7 @@ provision: | dev.provision ## This command will be deprecated in a future releas

stop: ## Stop all services
(test -d .docker-sync && docker-sync stop) || true ## Ignore failure here
docker-compose stop
docker-compose $(DOCKER_COMPOSE_FILES) stop

stop.watchers: ## Stop asset watchers
docker-compose $(DOCKER_COMPOSE_FILES) stop lms_watcher studio_watcher
Expand Down Expand Up @@ -263,96 +267,102 @@ pull: dev.pull
pull.xqueue: dev.pull.without-deps.xqueue+xqueue_consumer

validate: ## Validate the devstack configuration
docker-compose config
docker-compose $(DOCKER_COMPOSE_FILES) config

backup: dev.up.mysql+mongo+elasticsearch ## Write all data volumes to the host.
docker run --rm --volumes-from edx.devstack.mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql
docker run --rm --volumes-from edx.devstack.mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db
docker run --rm --volumes-from edx.devstack.elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data
backup: ## Write all data volumes to the host.
docker run --rm --volumes-from -T $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql
docker run --rm --volumes-from -T $$(make -s dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db
docker run --rm --volumes-from -T $$(make -s dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data

restore: dev.up.mysql+mongo+elasticsearch ## Restore all data volumes from the host. WARNING: THIS WILL OVERWRITE ALL EXISTING DATA!
docker run --rm --volumes-from edx.devstack.mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz
docker run --rm --volumes-from edx.devstack.mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz
docker run --rm --volumes-from edx.devstack.elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz
restore: ## Restore all data volumes from the host. WARNING: THIS WILL OVERWRITE ALL EXISTING DATA!
docker run --rm --volumes-from -T $$(make -s dev.print-container.mysql) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql.tar.gz
docker run --rm --volumes-from -T $$(make -s dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz
docker run --rm --volumes-from -T $$(make -s dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz

# TODO: Print out help for this target. Even better if we can iterate over the
# services in docker-compose.yml, and print the actual service names.
%-shell: ## Run a shell on the specified service container
docker exec -it edx.devstack.$* /bin/bash
docker-compose $(DOCKER_COMPOSE_FILES) exec $* /bin/bash

analyticspipeline-shell: ## Run a shell on the analytics pipeline container
docker exec -it edx.devstack.analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open
docker-compose $(DOCKER_COMPOSE_FILES) exec analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open

credentials-shell: ## Run a shell on the credentials container
docker exec -it edx.devstack.credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash'
docker-compose $(DOCKER_COMPOSE_FILES) exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash'

discovery-shell: ## Run a shell on the discovery container
docker exec -it edx.devstack.discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open
docker-compose $(DOCKER_COMPOSE_FILES) exec discovery env TERM=$(TERM) /edx/app/discovery/devstack.sh open

ecommerce-shell: ## Run a shell on the ecommerce container
docker exec -it edx.devstack.ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open
docker-compose $(DOCKER_COMPOSE_FILES) exec ecommerce env TERM=$(TERM) /edx/app/ecommerce/devstack.sh open

e2e-shell: ## Start the end-to-end tests container with a shell
docker run -it --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash
docker-compose run --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM) bash

registrar-shell: ## Run a shell on the registrar site container
docker exec -it edx.devstack.registrar env TERM=$(TERM) /edx/app/registrar/devstack.sh open
docker-compose $(DOCKER_COMPOSE_FILES) exec registrar env TERM=$(TERM) /edx/app/registrar/devstack.sh open

%-update-db: ## Run migrations for the specified service container
docker exec -t edx.devstack.$* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate'
docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate'

studio-update-db: ## Run migrations for the Studio container
docker exec -t edx.devstack.studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db'
docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db'

lms-update-db: ## Run migrations LMS container
docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db'
docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db'

update-db: | $(DB_MIGRATION_TARGETS) ## Run the migrations for DEFAULT_SERVICES

forum-restart-devserver: ## Kill the forum's Sinatra development server. The watcher process will restart it.
docker exec -t edx.devstack.forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")'
docker-compose $(DOCKER_COMPOSE_FILES) forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")'

lms-shell: ## Run a shell on the LMS container
docker exec -it edx.devstack.lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open
docker-compose $(DOCKER_COMPOSE_FILES) exec lms env TERM=$(TERM) /edx/app/edxapp/devstack.sh open

lms-watcher-shell: ## Run a shell on the LMS watcher container
docker exec -it edx.devstack.lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open
docker-compose $(DOCKER_COMPOSE_FILES) exec lms_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open

%-attach: ## Attach to the specified service container process to use the debugger & see logs.
docker attach edx.devstack.$*
docker attach "$$(make --silent dev.print-container.$*)"

lms-restart: lms-restart-devserver

studio-shell: ## Run a shell on the Studio container
docker exec -it edx.devstack.studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open
docker-compose $(DOCKER_COMPOSE_FILES) exec studio env TERM=$(TERM) /edx/app/edxapp/devstack.sh open

studio-watcher-shell: ## Run a shell on the studio watcher container
docker exec -it edx.devstack.studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open
docker-compose $(DOCKER_COMPOSE_FILES) exec studio_watcher env TERM=$(TERM) /edx/app/edxapp/devstack.sh open

studio-restart: studio-restart-devserver

xqueue-shell: ## Run a shell on the XQueue container
docker exec -it edx.devstack.xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open
docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue env TERM=$(TERM) /edx/app/xqueue/devstack.sh open

xqueue-restart: xqueue-restart-devserver

%-restart-devserver: ## Kill a service's Django development server. The watcher process should restart it.
docker exec -t edx.devstack.$* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")'
docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")'

xqueue_consumer-shell: ## Run a shell on the XQueue consumer container
docker exec -it edx.devstack.xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open
docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open

xqueue_consumer-restart: ## Kill the XQueue development server. The watcher process will restart it.
docker exec -t edx.devstack.xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")'
docker-compose $(DOCKER_COMPOSE_FILES) exec xqueue_consumer bash -c 'kill $$(ps aux | grep "manage.py run_consumer" | egrep -v "while|grep" | awk "{print \$$2}")'

%-static: ## Rebuild static assets for the specified service container
docker exec -t edx.devstack.$* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static'
docker-compose $(DOCKER_COMPOSE_FILES) exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static'

lms-sass: ## Rebuild sass assets for the LMS container
docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=lms --theme-dirs /edx/app/edx-themes/edx-platform/'

lms-static: ## Rebuild static assets for the LMS container
docker exec -t edx.devstack.lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets lms'
docker-compose $(DOCKER_COMPOSE_FILES) exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/'

studio-sass: ## Rebuild sass assets for the Studio container
docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver compile_sass --debug --system=cms --theme-dirs /edx/app/edx-themes/edx-platform/'

studio-static: ## Rebuild static assets for the Studio container
docker exec -t edx.devstack.studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets studio'
docker-compose $(DOCKER_COMPOSE_FILES) exec studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets --theme-dirs=/edx/app/edx-themes/edx-platform/'

static: | credentials-static discovery-static ecommerce-static lms-static studio-static ## Rebuild static assets for all service containers

Expand All @@ -365,24 +375,24 @@ e2e-tests: ## Run the end-to-end tests against the service containers

validate-lms-volume: ## Validate that changes to the local workspace are reflected in the LMS container
touch $(DEVSTACK_WORKSPACE)/edx-platform/testfile
docker exec edx.devstack.lms ls /edx/app/edxapp/edx-platform/testfile
docker-compose $(DOCKER_COMPOSE_FILES) exec -T lms ls /edx/app/edxapp/edx-platform/testfile
rm $(DEVSTACK_WORKSPACE)/edx-platform/testfile

vnc-passwords: ## Get the VNC passwords for the Chrome and Firefox Selenium containers
@docker logs edx.devstack.chrome 2>&1 | grep "VNC password" | tail -1
@docker logs edx.devstack.firefox 2>&1 | grep "VNC password" | tail -1
@docker-compose $(DOCKER_COMPOSE_FILES) logs chrome 2>&1 | grep "VNC password" | tail -1
@docker-compose $(DOCKER_COMPOSE_FILES) logs firefox 2>&1 | grep "VNC password" | tail -1

devpi-password: ## Get the root devpi password for the devpi container
docker-compose exec devpi bash -c "cat /data/server/.serverpassword"
docker-compose $(DOCKER_COMPOSE_FILES) exec devpi bash -c "cat /data/server/.serverpassword"

mysql-shell: ## Run a shell on the mysql container
docker-compose exec mysql bash
docker-compose $(DOCKER_COMPOSE_FILES) exec mysql bash

mysql-shell-edxapp: ## Run a mysql shell on the edxapp database
docker-compose exec mysql bash -c "mysql edxapp"
docker-compose $(DOCKER_COMPOSE_FILES) exec mysql bash -c "mysql edxapp"

mongo-shell: ## Run a shell on the mongo container
docker-compose exec mongo bash
docker-compose $(DOCKER_COMPOSE_FILES) exec mongo bash

dev.provision.analytics_pipeline: dev.provision.services.analyticspipeline

Expand All @@ -393,15 +403,15 @@ dev.up.analytics_pipeline: dev.up.analyticspipeline ## Bring up analytics pipeli
pull.analytics_pipeline: dev.pull.analyticspipeline ## Update analytics pipeline docker images

analytics-pipeline-devstack-test: ## Run analytics pipeline tests in travis build
docker exec -u hadoop -i edx.devstack.analytics_pipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity'
docker-compose $(DOCKER_COMPOSE_FILES) exec -u hadoop -T analytics_pipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity'

stop.analytics_pipeline: ## Stop all Analytics pipeline services.
docker-compose $(DOCKER_COMPOSE_FILES) stop \
namenode datanode resourcemanager nodemanager sparkmaster \
sparkworker vertica analyticspipeline

hadoop-application-logs-%: ## View hadoop logs by application Id
docker exec -it edx.devstack.analytics_pipeline.nodemanager yarn logs -applicationId $*
docker-compose $(DOCKER_COMPOSE_FILES) exec analytics_pipeline.nodemanager yarn logs -applicationId $*

# Provisions studio, ecommerce, and marketing with course(s) in test-course.json
# Modify test-course.json before running this make target to generate a custom course
Expand Down
10 changes: 5 additions & 5 deletions Makefile.edx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@ dev.provision.whitelabel:
# The containers must be started with the 'dev.up.e2e_wl_tests' target.
# AND the test must be setup using the 'dev.provision.whitelabel' target.
whitelabel-tests:
docker run -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e
docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh devstack.whitelabel:/tmp/run_whitelabel_tests.sh
docker exec -t devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh
docker-compose run -T -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e
docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh $(make --silent dev.print-container.devstack.whitelabel)":/tmp/run_whitelabel_tests.sh
docker-compose $(DOCKER_COMPOSE_FILES) exec devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh

whitelabel-cleanup:
docker rm --force devstack.whitelabel
docker-compose rm --force devstack.whitelabel

whitelabel-shell: ## Start a whitelabel test shell session
docker exec -it devstack.whitelabel env TERM=$(TERM) bash
docker-compose $(DOCKER_COMPOSE_FILES) exec -it devstack.whitelabel env TERM=$(TERM) bash
Loading