diff --git a/.github/workflows/add-depr-ticket-to-depr-board.yml b/.github/workflows/add-depr-ticket-to-depr-board.yml deleted file mode 100644 index 73ca4c5c6e87..000000000000 --- a/.github/workflows/add-depr-ticket-to-depr-board.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Run the workflow that adds new tickets that are either: -# - labelled "DEPR" -# - title starts with "[DEPR]" -# - body starts with "Proposal Date" (this is the first template field) -# to the org-wide DEPR project board - -name: Add newly created DEPR issues to the DEPR project board - -on: - issues: - types: [opened] - -jobs: - routeissue: - uses: openedx/.github/.github/workflows/add-depr-ticket-to-depr-board.yml@master - secrets: - GITHUB_APP_ID: ${{ secrets.GRAPHQL_AUTH_APP_ID }} - GITHUB_APP_PRIVATE_KEY: ${{ secrets.GRAPHQL_AUTH_APP_PEM }} - SLACK_BOT_TOKEN: ${{ secrets.SLACK_ISSUE_BOT_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/check-for-tutorial-prs.yml b/.github/workflows/check-for-tutorial-prs.yml deleted file mode 100644 index 6920542ac187..000000000000 --- a/.github/workflows/check-for-tutorial-prs.yml +++ /dev/null @@ -1,35 +0,0 @@ -# This workflow detects PRs that make changes to lms/templates/dashboard.html -# and only lms/templates/dashboard.html. This is the file that users are -# guided through changing in the Open edX tutorial: -# https://docs.openedx.org/en/latest/developers/quickstarts/first_openedx_pr.html#exercise-update-the-learner-dashboard - -# If this is the only file changed in the PR, we comment on the PR congratulating -# the user and letting others know that this is not a community PR in need of -# review. CODEOWNERS will tag a triaging team to provide reviews & ultimately -# close the PR. - -name: Check for Tutorial PR -description: Welcome contributors making their first PR from the tutorial -on: - pull_request: - types: [opened] - paths: - - 'lms/templates/dashboard.html' - -jobs: - # Provide helpful bot comment - comment: - runs-on: ubuntu-latest - name: provide helpful bot comment - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Comment PR - uses: thollander/actions-comment-pull-request@v1 - with: - message: | - Thank you for your pull request! Congratulations on completing the Open edX tutorial! A team member will be by to take a look shortly. - To those watching community pull requests: No need to worry about this one, a tCRIL team member will be taking care of it. - For this PR's author: If this is a PR that is NOT coming from the Open edX tutorial, please comment and let us know to disregard this message. - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml deleted file mode 100644 index 94b6b30600a8..000000000000 --- a/.github/workflows/ci-static-analysis.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Static analysis - -on: pull_request - -jobs: - tests: - name: Static analysis - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: ['3.8'] - os: ['ubuntu-20.04'] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system requirements - run: sudo apt update && sudo apt install -y libxmlsec1-dev - - - name: Install pip - run: python -m pip install -r requirements/pip.txt - - - name: Get pip cache dir - id: pip-cache-dir - run: echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install python dependencies - run: make dev-requirements - - - name: Static code analysis - run: make check-types diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml deleted file mode 100644 index fec11d6c259b..000000000000 --- a/.github/workflows/commitlint.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Run commitlint on the commit messages in a pull request. - -name: Lint Commit Messages - -on: - - pull_request - -jobs: - commitlint: - uses: openedx/.github/.github/workflows/commitlint.yml@master diff --git a/.github/workflows/docker-compose.yml.mysqldbdump b/.github/workflows/docker-compose.yml.mysqldbdump deleted file mode 100644 index 0853d250ff40..000000000000 --- a/.github/workflows/docker-compose.yml.mysqldbdump +++ /dev/null @@ -1,23 +0,0 @@ -version: '3' -services: - mysql: - image: mysql:5.7 - container_name: edx.devstack.mysql57 - ports: - - '3306:3306' - environment: - MYSQL_ROOT_PASSWORD: "" - MYSQL_ALLOW_EMPTY_PASSWORD: "yes" - volumes: - - ./init:/docker-entrypoint-initdb.d - healthcheck: - test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] - timeout: 20s - retries: 10 - edxapp: - image: edxops/edxapp:latest - command: bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' - volumes: - - ../../:/edx/app/edxapp/edx-platform - depends_on: - - mysql diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml deleted file mode 100644 index e7d9e0c1fa8e..000000000000 --- a/.github/workflows/docker-publish.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Push Docker Images - -on: - push: - branches: - - master -jobs: - # Push image to GitHub Packages. - # See also https://docs.docker.com/docker-hub/builds/ - push: - runs-on: ubuntu-latest - if: github.event_name == 'push' - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Build and Push docker image - env: - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - run : make docker_push diff --git a/.github/workflows/docs-build-check.yml b/.github/workflows/docs-build-check.yml deleted file mode 100644 index 10dd000ce0e2..000000000000 --- a/.github/workflows/docs-build-check.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Docs build - -on: - pull_request: - push: - branches: - - master - -jobs: - tests: - name: Docs build - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: ['3.8'] - os: ['ubuntu-20.04'] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system requirements - run: sudo apt update && sudo apt install -y libxmlsec1-dev - - - name: Install pip - run: python -m pip install -r requirements/pip.txt - - - name: Get pip cache dir - id: pip-cache-dir - run: echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install python dependencies - run: make dev-requirements - - - name: Install docs requirements - run: pip install -r requirements/edx/doc.txt - - - name: Docs build - run: make docs diff --git a/.github/workflows/init/01.sql b/.github/workflows/init/01.sql deleted file mode 100644 index 93d3a107e35e..000000000000 --- a/.github/workflows/init/01.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE DATABASE IF NOT EXISTS `edxapp`; -CREATE DATABASE IF NOT EXISTS `edxapp_csmh`; -GRANT ALL PRIVILEGES ON *.* TO 'edxapp001'@'%' IDENTIFIED BY 'password'; diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml deleted file mode 100644 index 743b5286d87e..000000000000 --- a/.github/workflows/js-tests.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: Javascript tests - -on: - pull_request: - branches: - - master - push: - branches: - - master - -jobs: - run_tests: - name: JS - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ ubuntu-20.04 ] - node-version: [ 16 ] - python-version: [ 3.8 ] - - steps: - - - uses: actions/checkout@v2 - - name: Fetch master to compare coverage - run: git fetch --depth=1 origin master - - - name: Setup Node - uses: actions/setup-node@v2 - with: - node-version: ${{ matrix.node-version }} - - - name: Setup npm - run: npm i -g npm@8.5.x - - - name: Install Firefox 61.0 - run: | - sudo apt-get purge firefox - wget "https://ftp.mozilla.org/pub/firefox/releases/61.0/linux-x86_64/en-US/firefox-61.0.tar.bz2" - tar -xjf firefox-61.0.tar.bz2 - sudo mv firefox /opt/firefox - sudo ln -s /opt/firefox/firefox /usr/bin/firefox - - - name: Install Required System Packages - run: sudo apt-get update && sudo apt-get install libxmlsec1-dev ubuntu-restricted-extras xvfb - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/base.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install Required Python Dependencies - run: | - make base-requirements - - - uses: c-hive/gha-npm-cache@v1 - - name: Run JS Tests - env: - TEST_SUITE: js-unit - SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh - run: | - npm install -g jest - xvfb-run --auto-servernum ./scripts/all-tests.sh - - - name: Save Job Artifacts - uses: actions/upload-artifact@v2 - with: - name: Build-Artifacts - path: | - reports/**/* - test_root/log/*.png - test_root/log/*.log - **/TEST-*.xml diff --git a/.github/workflows/lint-imports.yml b/.github/workflows/lint-imports.yml deleted file mode 100644 index 63caae452f3a..000000000000 --- a/.github/workflows/lint-imports.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Lint Python Imports - -on: - pull_request: - push: - branches: - - master - -jobs: - - lint-imports: - name: Lint Python Imports - runs-on: ubuntu-20.04 - - steps: - - name: Check out branch - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - - name: Install system requirements - run: sudo apt update && sudo apt install -y libxmlsec1-dev - - - name: Install pip - run: python -m pip install -r requirements/pip.txt - - - name: Get pip cache dir - id: pip-cache-dir - run: echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install python dependencies - run: pip install -r requirements/edx/development.txt - - # As long there are sub-projects[1] in edx-platform, we analyze each - # project separately here, in order to make import-linting errors easier - # to pinpoint. - # - # [1] https://openedx.atlassian.net/browse/BOM-2579 - - - name: Analyze imports (repo root) - run: make lint-imports diff --git a/.github/workflows/lockfileversion-check.yml b/.github/workflows/lockfileversion-check.yml deleted file mode 100644 index 42312e8cbf2d..000000000000 --- a/.github/workflows/lockfileversion-check.yml +++ /dev/null @@ -1,13 +0,0 @@ -#check package-lock file version - -name: Lockfile Version check - -on: - push: - branches: - - master - pull_request: - -jobs: - version-check: - uses: openedx/.github/.github/workflows/lockfileversion-check.yml@master diff --git a/.github/workflows/migrations-check-mysql8.yml b/.github/workflows/migrations-check-mysql8.yml deleted file mode 100644 index 74c4e6cabd84..000000000000 --- a/.github/workflows/migrations-check-mysql8.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Migrations check on MySql 8.0 - -on: - workflow_dispatch: - pull_request: - push: - branches: - - master - -jobs: - check_migrations: - name: check migrations mysql8 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ ubuntu-20.04 ] - python-version: [ 3.8 ] - - steps: - - name: Checkout repo - uses: actions/checkout@v2 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system Packages - run: | - sudo apt-get update - sudo apt-get install -y libxmlsec1-dev - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Ubuntu and sql Versions - run: | - lsb_release -a - mysql -V - - - name: Install Python dependencies - run: | - make dev-requirements - pip uninstall -y mysqlclient - pip install --no-binary mysqlclient mysqlclient - pip uninstall -y xmlsec - pip install --no-binary xmlsec xmlsec - - - name: Initiate Services - run: | - sudo systemctl start mongod - sudo /etc/init.d/mysql start - - - name: Reset mysql password - run: | - cat <> $GITHUB_PATH - - - name: Run Static Assets Check - env: - LMS_CFG: lms/envs/bok_choy.yml - CMS_CFG: cms/envs/bok_choy.yml - - run: | - paver update_assets lms - paver update_assets cms diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json deleted file mode 100644 index c250b00545c9..000000000000 --- a/.github/workflows/unit-test-shards.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "lms-1": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/badges/", - "lms/djangoapps/branding/", - "lms/djangoapps/bulk_email/", - "lms/djangoapps/bulk_enroll/", - "lms/djangoapps/bulk_user_retirement/", - "lms/djangoapps/ccx/", - "lms/djangoapps/certificates/", - "lms/djangoapps/commerce/" - ] - }, - "lms-2": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/course_api/", - "lms/djangoapps/course_blocks/", - "lms/djangoapps/course_goals/", - "lms/djangoapps/course_home_api/", - "lms/djangoapps/course_wiki/", - "lms/djangoapps/coursewarehistoryextended/", - "lms/djangoapps/debug/" - ] - }, - "lms-3": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/courseware/" - ] - }, - "lms-4": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/discussion/", - "lms/djangoapps/edxnotes/", - "lms/djangoapps/email_marketing/", - "lms/djangoapps/experiments/" - ] - }, - "lms-5": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/gating/", - "lms/djangoapps/grades/", - "lms/djangoapps/instructor/", - "lms/djangoapps/instructor_analytics/" - ] - }, - "lms-6": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/instructor_task/", - "lms/djangoapps/learner_dashboard/", - "lms/djangoapps/learner_home/", - "lms/djangoapps/learner_recommendations/", - "lms/djangoapps/lms_initialization/", - "lms/djangoapps/lms_xblock/", - "lms/djangoapps/lti_provider/", - "lms/djangoapps/mailing/", - "lms/djangoapps/mobile_api/", - "lms/djangoapps/monitoring/", - "lms/djangoapps/ora_staff_grader/", - "lms/djangoapps/program_enrollments/", - "lms/djangoapps/rss_proxy/", - "lms/djangoapps/save_for_later/", - "lms/djangoapps/static_template_view/", - "lms/djangoapps/staticbook/", - "lms/djangoapps/support/", - "lms/djangoapps/survey/", - "lms/djangoapps/teams/", - "lms/djangoapps/tests/", - "lms/djangoapps/user_tours/", - "lms/djangoapps/verify_student/", - "lms/djangoapps/mfe_config_api/", - "lms/envs/", - "lms/lib/", - "lms/tests.py" - ] - }, - "openedx-1": { - "settings": "lms.envs.test", - "paths": [ - "openedx/core/djangoapps/ace_common/", - "openedx/core/djangoapps/cors_csrf/", - "openedx/core/djangoapps/agreements/", - "openedx/core/djangoapps/api_admin/", - "openedx/core/djangoapps/auth_exchange/", - "openedx/core/djangoapps/bookmarks/", - "openedx/core/djangoapps/cache_toolbox/", - "openedx/core/djangoapps/catalog/", - "openedx/core/djangoapps/ccxcon/", - "openedx/core/djangoapps/commerce/", - "openedx/core/djangoapps/common_initialization/", - "openedx/core/djangoapps/common_views/", - "openedx/core/djangoapps/config_model_utils/", - "openedx/core/djangoapps/content/", - "openedx/core/djangoapps/content_libraries/", - "openedx/core/djangoapps/contentserver/", - "openedx/core/djangoapps/cookie_metadata/", - "openedx/core/djangoapps/course_apps/", - "openedx/core/djangoapps/course_date_signals/", - "openedx/core/djangoapps/course_groups/", - "openedx/core/djangoapps/courseware_api/", - "openedx/core/djangoapps/crawlers/", - "openedx/core/djangoapps/credentials/", - "openedx/core/djangoapps/credit/", - "openedx/core/djangoapps/course_live/", - "openedx/core/djangoapps/dark_lang/", - "openedx/core/djangoapps/debug/", - "openedx/core/djangoapps/demographics/", - "openedx/core/djangoapps/discussions/", - "openedx/core/djangoapps/django_comment_common/", - "openedx/core/djangoapps/embargo/", - "openedx/core/djangoapps/enrollments/", - "openedx/core/djangoapps/external_user_ids/" - ] - }, - "openedx-2": { - "settings": "lms.envs.test", - "paths": [ - "openedx/core/djangoapps/geoinfo/", - "openedx/core/djangoapps/header_control/", - "openedx/core/djangoapps/heartbeat/", - "openedx/core/djangoapps/lang_pref/", - "openedx/core/djangoapps/models/", - "openedx/core/djangoapps/monkey_patch/", - "openedx/core/djangoapps/oauth_dispatch/", - "openedx/core/djangoapps/olx_rest_api/", - "openedx/core/djangoapps/password_policy/", - "openedx/core/djangoapps/plugin_api/", - "openedx/core/djangoapps/plugins/", - "openedx/core/djangoapps/profile_images/", - "openedx/core/djangoapps/programs/", - "openedx/core/djangoapps/safe_sessions/", - "openedx/core/djangoapps/schedules/", - "openedx/core/djangoapps/service_status/", - "openedx/core/djangoapps/session_inactivity_timeout/", - "openedx/core/djangoapps/signals/", - "openedx/core/djangoapps/site_configuration/", - "openedx/core/djangoapps/system_wide_roles/", - "openedx/core/djangoapps/theming/", - "openedx/core/djangoapps/user_api/", - "openedx/core/djangoapps/user_authn/", - "openedx/core/djangoapps/util/", - "openedx/core/djangoapps/verified_track_content/", - "openedx/core/djangoapps/video_config/", - "openedx/core/djangoapps/video_pipeline/", - "openedx/core/djangoapps/waffle_utils/", - "openedx/core/djangoapps/xblock/", - "openedx/core/djangoapps/xmodule_django/", - "openedx/core/djangoapps/zendesk_proxy/", - "openedx/core/djangolib/", - "openedx/core/lib/", - "openedx/core/tests/", - "openedx/features/", - "openedx/testing/", - "openedx/tests/" - ] - }, - "openedx-3": { - "settings": "cms.envs.test", - "paths": [ - "openedx/core/djangoapps/ace_common/", - "openedx/core/djangoapps/cors_csrf/", - "openedx/core/djangoapps/agreements/", - "openedx/core/djangoapps/api_admin/", - "openedx/core/djangoapps/auth_exchange/", - "openedx/core/djangoapps/bookmarks/", - "openedx/core/djangoapps/cache_toolbox/", - "openedx/core/djangoapps/catalog/", - "openedx/core/djangoapps/ccxcon/", - "openedx/core/djangoapps/commerce/", - "openedx/core/djangoapps/common_initialization/", - "openedx/core/djangoapps/common_views/", - "openedx/core/djangoapps/config_model_utils/", - "openedx/core/djangoapps/content/", - "openedx/core/djangoapps/content_libraries/", - "openedx/core/djangoapps/contentserver/", - "openedx/core/djangoapps/cookie_metadata/", - "openedx/core/djangoapps/course_apps/", - "openedx/core/djangoapps/course_date_signals/", - "openedx/core/djangoapps/course_groups/", - "openedx/core/djangoapps/courseware_api/", - "openedx/core/djangoapps/crawlers/", - "openedx/core/djangoapps/credentials/", - "openedx/core/djangoapps/credit/", - "openedx/core/djangoapps/dark_lang/", - "openedx/core/djangoapps/debug/", - "openedx/core/djangoapps/demographics/", - "openedx/core/djangoapps/discussions/", - "openedx/core/djangoapps/django_comment_common/", - "openedx/core/djangoapps/embargo/", - "openedx/core/djangoapps/enrollments/", - "openedx/core/djangoapps/external_user_ids/" - ] - }, - "openedx-4": { - "settings": "cms.envs.test", - "paths": [ - "openedx/core/djangoapps/geoinfo/", - "openedx/core/djangoapps/header_control/", - "openedx/core/djangoapps/heartbeat/", - "openedx/core/djangoapps/lang_pref/", - "openedx/core/djangoapps/models/", - "openedx/core/djangoapps/monkey_patch/", - "openedx/core/djangoapps/oauth_dispatch/", - "openedx/core/djangoapps/olx_rest_api/", - "openedx/core/djangoapps/password_policy/", - "openedx/core/djangoapps/plugin_api/", - "openedx/core/djangoapps/plugins/", - "openedx/core/djangoapps/profile_images/", - "openedx/core/djangoapps/programs/", - "openedx/core/djangoapps/safe_sessions/", - "openedx/core/djangoapps/schedules/", - "openedx/core/djangoapps/service_status/", - "openedx/core/djangoapps/session_inactivity_timeout/", - "openedx/core/djangoapps/signals/", - "openedx/core/djangoapps/site_configuration/", - "openedx/core/djangoapps/system_wide_roles/", - "openedx/core/djangoapps/theming/", - "openedx/core/djangoapps/user_api/", - "openedx/core/djangoapps/user_authn/", - "openedx/core/djangoapps/util/", - "openedx/core/djangoapps/verified_track_content/", - "openedx/core/djangoapps/video_config/", - "openedx/core/djangoapps/video_pipeline/", - "openedx/core/djangoapps/waffle_utils/", - "openedx/core/djangoapps/xblock/", - "openedx/core/djangoapps/xmodule_django/", - "openedx/core/djangoapps/zendesk_proxy/", - "openedx/core/lib/", - "openedx/tests/" - ] - }, - "cms-1": { - "settings": "cms.envs.test", - "paths": [ - "cms/djangoapps/api/", - "cms/djangoapps/cms_user_tasks/", - "cms/djangoapps/coursegraph/", - "cms/djangoapps/course_creators/", - "cms/djangoapps/export_course_metadata/", - "cms/djangoapps/maintenance/", - "cms/djangoapps/models/", - "cms/djangoapps/pipeline_js/", - "cms/djangoapps/xblock_config/", - "cms/envs/", - "cms/lib/" - ] - }, - "cms-2": { - "settings": "cms.envs.test", - "paths": [ - "cms/djangoapps/contentstore/" - ] - }, - "common-1": { - "settings": "lms.envs.test", - "paths": [ - "common/djangoapps/" - ] - }, - "common-2": { - "settings": "cms.envs.test", - "paths": [ - "common/djangoapps/" - ] - }, - "xmodule-1": { - "settings": "lms.envs.test", - "paths": [ - "xmodule/" - ] - } -} diff --git a/.github/workflows/unit-tests-gh-hosted.yml b/.github/workflows/unit-tests-gh-hosted.yml deleted file mode 100644 index fc5f9bee6371..000000000000 --- a/.github/workflows/unit-tests-gh-hosted.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: unit-tests-gh-hosted - -on: - pull_request: - push: - branches: - - master - - open-release/lilac.master - -jobs: - run-test: - if: (github.repository != 'openedx/edx-platform' && github.repository != 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == true)) - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - python-version: [ '3.8' ] - django-version: - - "pinned" - shard_name: [ - "lms-1", - "lms-2", - "lms-3", - "lms-4", - "lms-5", - "lms-6", - "openedx-1", - "openedx-2", - "openedx-3", - "openedx-4", - "cms-1", - "cms-2", - "common-1", - "common-2", - "common-3", - ] - name: gh-hosted-python-${{ matrix.python-version }},django-${{ matrix.django-version }},${{ matrix.shard_name }} - steps: - - uses: actions/checkout@v2 - - - name: Install Required System Packages - run: sudo apt-get update && sudo apt-get install libxmlsec1-dev lynx - - - name: Start MongoDB - uses: supercharge/mongodb-github-action@1.7.0 - with: - mongodb-version: 4.4 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install Required Python Dependencies - env: - PIP_SRC: ${{ runner.temp }} - run: | - make test-requirements - if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - pip install "django~=${{ matrix.django-version }}.0" - pip check # fail if this test-reqs/Django combination is broken - fi - - - name: Setup and run tests - uses: ./.github/actions/unit-tests - - collect-and-verify: - if: (github.repository != 'openedx/edx-platform' && github.repository != 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == true)) - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: [ '3.8' ] - django-version: - - "pinned" - steps: - - uses: actions/checkout@v2 - - - name: Install Required System Packages - run: sudo apt-get update && sudo apt-get install libxmlsec1-dev - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install Required Python Dependencies - run: | - make test-requirements - if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - pip install "django~=${{ matrix.django-version }}.0" - pip check # fail if this test-reqs/Django combination is broken - fi - - - name: verify unit tests count - uses: ./.github/actions/verify-tests-count diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml deleted file mode 100644 index 69a03bbeafff..000000000000 --- a/.github/workflows/unit-tests.yml +++ /dev/null @@ -1,153 +0,0 @@ -name: unit-tests - -on: - pull_request: - push: - branches: - - master - -jobs: - run-tests: - name: python-${{ matrix.python-version }},django-${{ matrix.django-version }},${{ matrix.shard_name }} - if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) - runs-on: [ edx-platform-runner ] - strategy: - matrix: - python-version: - - "3.8" - django-version: - - "pinned" - #- "4.0" - shard_name: - - "lms-1" - - "lms-2" - - "lms-3" - - "lms-4" - - "lms-5" - - "lms-6" - - "openedx-1" - - "openedx-2" - - "openedx-3" - - "openedx-4" - - "cms-1" - - "cms-2" - - "common-1" - - "common-2" - - "xmodule-1" - # We expect Django 4.0 to fail, so don't stop when it fails. - continue-on-error: ${{ matrix.django-version == '4.0' }} - - steps: - - name: sync directory owner - run: sudo chown runner:runner -R .* - - - name: checkout repo - uses: actions/checkout@v3 - - - name: start mongod server for tests - run: | - sudo mkdir -p /data/db - sudo chmod -R a+rw /data/db - mongod & - - - name: install requirements - run: | - sudo make test-requirements - if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - sudo pip install "django~=${{ matrix.django-version }}.0" - sudo pip check # fail if this test-reqs/Django combination is broken - fi - - - name: list installed package versions - run: | - sudo pip freeze - - - name: Setup and run tests - uses: ./.github/actions/unit-tests - - - name: Renaming coverage data file - run: | - mv reports/.coverage reports/${{ matrix.shard_name }}.coverage - - - name: Upload coverage - uses: actions/upload-artifact@v3 - with: - name: coverage - path: reports/${{matrix.shard_name}}.coverage - - # This job aggregates test results. It's the required check for branch protection. - # https://github.com/marketplace/actions/alls-green#why - # https://github.com/orgs/community/discussions/33579 - success: - name: Tests successful - if: always() - needs: - - run-tests - runs-on: ubuntu-latest - steps: - - name: Decide whether the needed jobs succeeded or failed - # uses: re-actors/alls-green@v1.2.1 - uses: re-actors/alls-green@13b4244b312e8a314951e03958a2f91519a6a3c9 - with: - jobs: ${{ toJSON(needs) }} - - compile-warnings-report: - runs-on: [ edx-platform-runner ] - needs: [ run-tests ] - steps: - - name: sync directory owner - run: sudo chown runner:runner -R .* - - uses: actions/checkout@v3 - - name: collect pytest warnings files - uses: actions/download-artifact@v2 - with: - name: pytest-warnings-json - path: test_root/log - - - name: display structure of downloaded files - run: ls -la test_root/log - - - name: compile warnings report - run: | - python openedx/core/process_warnings.py --dir-path test_root/log --html-path reports/pytest_warnings/warning_report_all.html - - - name: save warning report - if: success() - uses: actions/upload-artifact@v3 - with: - name: pytest-warning-report-html - path: | - reports/pytest_warnings/warning_report_all.html - - # Combine and upload coverage reports. - coverage: - needs: run-tests - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [ 3.8 ] - steps: - - name: Checkout repo - uses: actions/checkout@v3 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Download all artifacts - uses: actions/download-artifact@v3 - with: - name: coverage - path: reports - - - name: Install Python dependencies - run: | - pip install -r requirements/edx/coverage.txt - - - name: Run coverage - run: | - coverage combine reports/* - coverage report - coverage xml - - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml deleted file mode 100644 index 911c6f0e51da..000000000000 --- a/.github/workflows/upgrade-python-requirements.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Upgrade Requirements - -on: - schedule: - - cron: "0 2 * * 2" - workflow_dispatch: - inputs: - branch: - description: 'Target branch to create requirements PR against' - required: true - default: 'master' -jobs: - call-upgrade-python-requirements-workflow: - with: - branch: ${{ github.event.inputs.branch }} - team_reviewers: "arbi-bom" - email_address: arbi-bom@edx.org - send_success_notification: false - secrets: - requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }} - requirements_bot_github_email: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }} - edx_smtp_username: ${{ secrets.EDX_SMTP_USERNAME }} - edx_smtp_password: ${{ secrets.EDX_SMTP_PASSWORD }} - uses: openedx/.github/.github/workflows/upgrade-python-requirements.yml@master - diff --git a/.github/workflows/verify-dunder-init.yml b/.github/workflows/verify-dunder-init.yml deleted file mode 100644 index aefc0f53b6f5..000000000000 --- a/.github/workflows/verify-dunder-init.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: CI - -on: - pull_request: - branches: - - master - -jobs: - - verify_dunder_init: - - name: Verify __init__.py Files - runs-on: ubuntu-20.04 - - steps: - - - name: Check out branch - uses: actions/checkout@v2 - - - name: Ensure git is installed - run: | - sudo apt-get update && sudo apt-get install git - - - name: Verify __init__.py files exist - run: | - scripts/verify-dunder-init.sh diff --git a/.github/workflows/verify-gha-unit-tests-count.yml b/.github/workflows/verify-gha-unit-tests-count.yml deleted file mode 100644 index c68092942d70..000000000000 --- a/.github/workflows/verify-gha-unit-tests-count.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: verify unit tests count - -on: - pull_request: - push: - branches: - - master - -jobs: - collect-and-verify: - if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) - runs-on: [ edx-platform-runner ] - steps: - - name: sync directory owner - run: sudo chown runner:runner -R .* - - - uses: actions/checkout@v2 - - name: install requirements - run: | - sudo make test-requirements - - - name: verify unit tests count - uses: ./.github/actions/verify-tests-count diff --git a/docs/decisions/0017-reimplement-asset-processing.rst b/docs/decisions/0017-reimplement-asset-processing.rst new file mode 100644 index 000000000000..eef472a0ff28 --- /dev/null +++ b/docs/decisions/0017-reimplement-asset-processing.rst @@ -0,0 +1,269 @@ +Reimplement edx-platform static asset processing +################################################ + +Overview +******** + +* edx-platform has a complicated process for managing its static frontend assets. It slows down both developers and site operators. +* We will deprecate the current Python+paver asset processing system in favor of a new Bash implementation. +* After one named release, the deprecated paver system will be removed. + +Status +****** + +Provisional + +Will be moved to *Accepted* upon completion of reimplementation. + +Non-exhaustive list of related PRs: + +* https://github.com/openedx/edx-platform/pull/31736 +* ... + +Context +******* + +State of edx-platform frontends +=============================== + +New Open edX frontend development has largely moved to React-based micro-frontends (MFEs). However, edx-platform still has a few categories of important static frontend assets: + +.. list-table:: + :header-rows: 1 + + * - **Name** + - Description + - Example + - Expected direction + * - **Legacy LMS Frontends** + - JS, SCSS, and other resources powering LMS views that have not yet been replatformed into MFEs + - Instructor Dashboard assets + - Replatform & DEPR + * - **Legacy CMS Frontends** + - JS, SCSS, and other resources powering Studio views that have not yet been replatformed into MFEs + - Course outline editor and unit editor assets + - Replatform & DEPR + * - **Shared Frontend Files** + - JS modules, SCSS partials, and other resources, usable by both Legacy LMS and CMS Frontends. This includes a few vendor libraries that have been committed to edx-platform in their entirety. + - Legacy cookie policy banner; CodeMirror + - Remove as part of full LMS/CMS frontend replatforming + * - **npm-installed Assets** + - JS modules and CSS files installed via NPM. Not committed to edx-platform. + - React, studio-frontend, paragon + - Uninstall as part of full LMS/CMS frontend replatforming + * - **XModule Fragments** + - JS and SCSS belonging to the older XModule-style XBlocks defined in edx-platform + - ProblemBlock (aka CAPA) assets + - Convert to pure XBlock fragments + * - **XBlock Fragments** + - JS and CSS belonging to the pure XBlocks defined in edx-platform + - library_sourced_block.js + - Keep and/or extract to pip-installed, per-XBlock repositories + * - **pip-installed Assets** + - Pre-compiled static assets shipped with several Python libraries that we install, including XBlocks. Not committed to edx-platform. + - Django Admin, Swagger, Drag-And-Drop XBlock V2 + - Keep + +*Note: this table excludes HTML templates. Templates are part of the frontend, but they are dynamically rendered by the Web application and therefore must be handled differently than static assets.* + +So, with the exception of XBlock fragments and pip-installed assets, which are very simple for edx-platform to handle, we plan to eventually remove all edx-platform static frontend assets. However, given the number of remaining edx-platform frontends and speed at which they are currently being replatformed, estimates for completion of this process range from one to five years. Thus, in the medium term future, we feel that timeboxed improvements to how edx-platform handles static assets are worthwhile, especially when they address an acute pain point. + +Current pain points +=================== + +Three particular issues have surfaced in Developer Experience Working Group discussions recently, each with some mitigations involving static assets: + +.. list-table:: + :header-rows: 1 + + * - Pain Point + - Potential solution(s) + + * - edx-platform Docker images are too large and/or take too long to build. + - Switch from large, legacy tooling packages (such as libsass-python and paver) to industry standard, pre-compiled ones (like node-sass or dart-sass). Remove unnecessary & slow calls to Django management commands. + + * - edx-platform Docker image layers seem to be rebuilt more often than they should. + - Remove all Python dependencies from the static asset build process, such that changes to Python code or requirements do not always have to result in a static asset rebuild. + + * - In Tutor, using a local copy of edx-platform overwrites the Docker image's pre-installed node_modules and pre-built static assets, requiring developers to reinstall & rebuild in order to get a working platform. + - Better parameterize the input and output paths edx-platform asset build, such that it may `search for node_modules outside of edx-platform and `generate assets outside of edx-platform `. + +All of these potential solutions would involve refactoring or entirely replacing parts of the current asset processing system. + + +Decision +******** + +We will largely reimplement edx-platform's asset processing system. We will aim to: + +* Use well-known, npm-installed frontend tooling wherever possible. +* When bespoke processing is required, use standard POSIX tools like Bash. +* When Django/Python is absolutely required, contain its impact so that the rest of the system remains Python-free. +* Avoid unnecessary indirection or abstraction. For this task, extensibility is a non-goal, and simplicity is a virtue. +* Provide a clear migration path from the old system to the new one. +* Enable the future removal of as much legacy frontend tooling code as possible. + +Consequences +************ + +Reimplementation Specification +============================== + +The three top-level edx-platform asset processing actions are *build*, *collect*, and *watch*. The build action can be further broken down into five stages. Here is how those actions and stages will be reimplemented: + + +.. list-table:: + :header-rows: 1 + + * - Description + - Old implementation + - New implementation + + * - **Build: All stages.** Compile, generate, copy, and otherwise process static assets so that they can be used by the Django webserver or collected elsewhere. For many Web applications, all static asset building would be coordinated via Webpack or another NPM-managed tool. Due to the age of edx-platform and its legacy XModule and Comprehensive Theming systems, though, there are five stages which need to be performed in a particular order. + + - ``paver update_assets --skip-collect`` + + A Python-defined task that calls out to each build stage. + + - ``assets/build.sh`` + + A Bash script that contains all build stages, with subcommands available for running each stage separately. Its command-line interface inspired by Tutor's ``openedx-assets`` script. The script will be runnable on any POSIX system, including macOS and Ubuntu and it will linted for common shell scripting mistakes using `shellcheck `_. + + * - + **Build stage 1: Copy npm-installed assets** from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. + + - ``paver update_assets --skip-collect`` + + Implemented in Python within update_assets. There is no standalone command for it. + + - ``assets/build.sh npm`` + + Pure Bash reimplementation. + + * - + **Build stage 2: Copy XModule fragments** from the xmodule source tree over to input directories for Webpack and SCSS compilation. This is required for a hard-coded list of old XModule-style XBlocks. This is not required for new pure XBlocks, which include (or pip-install) their assets into edx-platform as ready-to-serve JS/CSS/etc fragments. + + - ``paver process_xmodule_assets``, or + + ``xmodule_assets`` + + Equivalent paver task and console script, both pointing at to an application-level Python module. That module inspects attributes from legacy XModule-style XBlock classes in order to determine which static assets to copy and what to name them. + + - ``assets/build.sh xmodule`` + + A Bash implementation of XModule asset copying. The aforementioned attributes will be moved from the XModule-style XBlock classes into a simple static JSON file, which the Bash script will be able to read. + + The initial implementation of build.sh may just point at ``xmodule_assets``. + + * - + **Build stage 3: Run Webpack** in order to to shim, minify, otherwise process, and bundle JS modules. This requires a call to the npm-installed ``webpack`` binary. + + - ``paver webpack`` + + Python wrapper around a call to webpack. Invokes the ``./manage.py [lms|cms] print_setting`` multiple times in order to determine Django settings, adding which can add 20+ seconds to the build. + + - ``assets/build.sh webpack`` + + Bash wrapper around a call to webpack. The script will accept parameters for Django settings rather than looking them up. Open edX distributions, such as Tutor, can choose how to supply the Django-setting-derived parameters in an efficient manner. + + * - + **Build stage 4: Compile default SCSS** into CSS for legacy LMS/CMS frontends. + + - ``paver compile_sass`` + + Paver task that invokes ``sass.compile`` (from the libsass Python package) and ``rtlcss`` (installed by npm) for several different directories of SCSS. + + Note: libsass is pinned to a 2015 version with a non-trivial upgrade path. Installing it requires compiling a large C extension, noticeably affecting Docker image build time. + + - ``assets/build.sh common`` + + Bash reimplementation, calling ``node-sass`` and ``rtlcss``. + + The initial implementation of build.sh may use ``sassc``, a CLI provided by libsass, instead of node-sass. Then, ``sassc`` can be replaced by ``node-sass`` as part of a subsequent `edx-platform frontend framework upgrade effort `_. + + * - + **Build stage 5: Compiled themes' SCSS** into CSS for legacy LMS/CMS frontends. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. + + - ``./manage.py [lms|cms] compile_sass``, or + + ``paver compile_sass --theme-dirs ...`` + + The management command is a wrapper around the paver task. The former looks up the list of theme search directories from Django settings and site configuration; the latter requires them to be supplied as arguments. + + - ``./manage.py [lms|cms] compile_sass``, or + + ``assets/build.sh themes --theme-dirs ...`` + + The management command will remain available, but it will need to be updated to point at the Bash script, which will replace the paver task (see build stage 4 for details). + + The overall asset *build* action will use the Bash script; this means that list of theme directories will need to be provided as arguments, but it ensures that the build can remain Python-free. + + * - **Collect** the built static assets from edx-platform to another location (the ``STATIC_ROOT``) so that they can be efficiently served *without* Django's webserver. This step, by nature, requires Python and Django in order to find and organize the assets, which may come from edx-platform itself or from its many installed Python and NPM packages. This is only needed for **production** environments, where it is usually desirable to serve assets with something efficient like NGINX. + + - ``paver update_assets`` + + Paver task wrapping a call to the standard Django `collectstatic `_ command. It adds ``--noinput`` and a list of ``--ignore`` file patterns to the command call. + + (This command also builds assets. The *collect* action could not be run on its own without calling pavelib's Python interface.) + + - ``./manage.py lms collectstatic --noinput && ./manage.py cms collectstatic --noinput`` + + The standard Django interface will be used without a wrapper. The ignore patterns will be added to edx-platform's `staticfiles app configuration `_ so that they do not need to be supplied as part of the command. + + * - **Watch** static assets for changes in the background. When a change occurs, rebuild them automatically, so that the Django webserver picks up the changes. This is only necessary in **development** environments. A few different sets of assets may be watched: XModule fragments, Webpack assets, default SCSS, and theme SCSS. + + - ``paver watch_assets`` + + Paver task that invokes ``webpack --watch`` for Webpack assets and watchdog (a Python library) for other assets. + + - ``assets/build.sh --watch `` + + (where ```` is optionally one of the build stages described above. If provided, only that stage's assets will be watched.) + + Bash wrappers around invocation(s) of `watchman `_, a popular file-watching library maintained by Meta. Watchman is already installed into edx-platform (and other services) via the pywatchman pip wrapper package. + + Note: This adds a Python dependency to build.sh. However, we could be clear that watchman is an *optional* dependency of build.sh which enables the optional ``--watch`` feature. This would keep the *build* action Python-free. Alternatively, watchman is also available Python-free via apt and homebrew. + +Migration +========= + +The old asset processing system will be `proposed for deprecation (TODO: link to issue) `_ upon provisional acceptance of this ADR. + +The old and new systems will both be available for at least one named release. Operators will encouraged to try the new asset processing system and report any issues they find. Eventually, the old asset processing system will be entirely removed. + +Tutor migration guide +--------------------- + +Tutor provides the `openedx-assets `_ Python script on its edx-platform images for building, collection, and watching. The script uses a mix its own implementation and calls out to edx-platform's paver tasks, avoiding the most troublesome parts of the paver tasks. The script and its interface were the inspiration for the new build.sh that this ADAR describes. + +As a consequence of this ADR, Tutor will either need to: + +* reimplement the script as a thin wrapper around the new asset processing commands, or +* deprecate and remove the script. + +Either way, the migration path is straightforward: + +.. list-table:: + :header-rows: 1 + + * - Existing Tutor-provided command + - New upstream command + * - ``openedx-assets build`` + - ``assets/build.sh`` + * - ``openedx-assets npm`` + - ``assets/build.sh npm`` + * - ``openedx-assets xmodule`` + - ``assets/build.sh xmodule`` + * - ``openedx-assets common`` + - ``assets/build.sh common`` + * - ``openedx-assets themes`` + - ``assets/build.sh`` + * - ``openedx-assets collect`` + - ``./manage.py [lms|cms] collectstatic --noinput`` + * - ``openedx-assets watch-themes`` + - ``assets/build.sh --watch themes`` + +The options accepted by ``openedx-assets`` will all be valid inputs to ``assets/build.sh``. + +Rejected Alternatives +********************* + +* **Avoiding work on edx-platform asset tooling; just wait until all frontends have been replatformed into MFEs**. See Context for why this was rejected. + +* **Rather than replace paver-based asset processing, try to improve it in place.** The effort required to do this seemed comparable to the effort required to perform a full rewrite, and it would not yield any caching benefits of a Python-free asset pipeline. + diff --git a/scripts/assets.sh b/scripts/assets.sh new file mode 100755 index 000000000000..08d3ee62bffb --- /dev/null +++ b/scripts/assets.sh @@ -0,0 +1,694 @@ +#!/usr/bin/env bash +ABOUT="Various assets processing/building/collection utility for Open edX" + +# Enable stricter error handling. +set -euo pipefail + +# True script name, so that we can call ourselves from watchers +THIS_SCRIPT="$0" + +# Constants, overridable via environment variables +SCRIPT_NAME="${SCRIPT_NAME:-$THIS_SCRIPT}" # Script name for logging & errors +DEFAULT_STATIC_ROOT="${DEFAULT_STATIC_ROOT:-./test_root/static}" # Fallback STATIC_ROOT (for Django settings) +DEFAULT_THEMES_DIR="${DEFAULT_THEMES_DIR:-}" # Fallback directory to look for themes in +DEFAULT_COLLECT_SETTINGS="${DEFAULT_COLLECT_SETTINGS:-lms.envs.production}" # Fallback `./manage.py collectstatic` settings + +# Codes for colored terminal output. +COL_LOG="\e[36m" # Log/step/section color (cyan) +COL_RUN="\e[35m" # Executed code echo color (purple) +COL_ERR="\e[31m" # Error color (red) +COL_OFF="\e[0m" # Normal color + +# Print script usage information. +show_usage ( ) { + echo "Usage: $SCRIPT_NAME [] []" + echo + echo "$ABOUT" + echo + echo "Subcommands:" + echo " build Build all assets (npm+xmodule+webpack+common+themes)" + echo " npm Copy static assets from node_modules" + echo " xmodule Process assets from xmodule" + echo " webpack Run webpack" + echo " common Compile static assets for common theme" + echo " themes Compile static assets for custom themes" + echo " collect Collect static assets to be served by webserver" + echo " watch-themes Watch theme assets for changes and recompile on-the-fly" + echo + echo "Options:" + echo " -h|--help Display this help message." + echo " -d|--dry-run Print shell commands but do not run them." + echo " -e|--env Environment: prod or dev. Default is prod." + echo " -r|--static-root Path for Webpack output." + echo " Default is $DEFAULT_STATIC_ROOT." + echo " -s|--settings Dotted path to Django settings module for asset" + echo " collection. Default is $DEFAULT_COLLECT_SETTINGS" + echo " --systems ... Specify one or more systems: lms, cms. Default is both." + echo " --theme-dirs ... Specify one or more theme search dirs." + echo " Default is ${DEFAULT_THEMES_DIR:-none}." + echo " --themes ... Compile one or more themes from theme search dirs." + echo " Default is all." +} + +# Print a formatted error message. +show_error ( ) { + local error_message="$1" + + echo -e "${COL_ERR}${SCRIPT_NAME}: error: ${error_message}${COL_OFF}" +} + +# Print a formatted error message, and exit the script unsuccessfully. +fail ( ) { + local error_message="$1" + + show_error "$error_message" + exit 1 +} + +# Print a formatted error message, tell the user how to view the script's usage info, +# and exit the script unsucessfully. Use this function when the error is simply +# that the script has been called wrong. +fail_usage ( ) { + local error_message="$1" + + show_error "$error_message" + echo + echo "Try '$SCRIPT_NAME -h' or '$SCRIPT_NAME --help' for more information." + exit 1 +} + +# Log the beginning of a "section" (a larger part of the script). +log_section_start ( ) { + local section_description="$*" + + echo -e "${COL_LOG}=====================================================================================$COL_OFF" + echo -e "${COL_LOG} $section_description $COL_OFF" + echo -e "${COL_LOG}-------------------------------------------------------------------------------$COL_OFF" +} + +# Log the end of a "section" (a larger part of the script). +log_section_end ( ) { + local section_description="$*" + + echo -e "${COL_LOG}-------------------------------------------------------------------------------$COL_OFF" + echo -e "${COL_LOG} $section_description $COL_OFF" + echo -e "${COL_LOG}=====================================================================================$COL_OFF" +} + +# Log a line of information. +log ( ) { + local log_line="$*" + + echo -e "${COL_LOG}$SCRIPT_NAME: $log_line $COL_OFF" +} + +# run: a 'function pointer' to either _echo_and_run_command or _echo_command. +# +# All shell commands invoked by this script that mutate the environment +# are wrapped in a call to '"$run" ...'. This gives us two benefits: +# 1. Commands are always echoed before they are executed, making it +# easier for users to understand & debug the script from its output. +# 2. If the user passses --dry-run, we set run="_echo_command", +# allowing to the script to be run with command printed but not executed. +run="_echo_and_run_command" + +# Print a shell command (without running it). +# Command should be passed as separate arguments. +_echo_command ( ) { + local command_components=("$@") + + echo -e "${COL_RUN}${command_components[*]}${COL_OFF}" +} + +# Print & execute a shell command. +# Command should be passed as separate arguments. +_echo_and_run_command ( ) { + local command_components=("$@") + + _echo_command "${command_components[@]}" + "${command_components[@]}" +} + +# Compile a directory of SCSS into CSS, generating RTL CSS as needed. +# +# TODO: Unlike its Python API, libsass-python's CLI (sassc) only supports compiling individual +# SCSS files, not entire directories. However, the CLIs for dart-sass and node-sass +# both *do* support compiling entire directories. So, if/when we upgrade to one of those +# libraries, much of this function can be replaced with a single CLI call. +compile_scss_dir ( ) { + local scss_env="$1" # 1: Environment (dev or prod). For output styling. + local scss_src_root="$2" # 2: Path to source directory containing SCSS. + local css_dest_root="$3" # 3: Path to target directory for CSS. + shift 3 + local include_paths=("$@") # Remaining args: Search paths for SCSS imports. + + local sassc_options=() + + # Add output-style option, depending on environment. + if [[ "$scss_env" == dev ]] ; then + sassc_options+=("--output-style=nested") + else + sassc_options+=("--output-style=compressed") + fi + + # For each include path, add it to the list of SCSS compile options. + for include_path in "${include_paths[@]}" ; do + sassc_options+=("--include-path=$include_path") + done + + # For each SCSS file $scss_src within $scss_src_root (recursive), + # excluding underscore-prefixed (i.e., partial) SCSS files... + while read -r -d $'\0' scss_src ; do + + # Translate source path into destination path: + scss_src_relative="${scss_src#"$scss_src_root"}" # Chop off SCSS root dir prefix. + css_dest_relative="${scss_src_relative%.scss}.css" # Replace file extension. + css_dest="$css_dest_root/$css_dest_relative" # Prepend CSS root dir. + + css_dest_dir="$(dirname "$css_dest")" # Find immediate parent dir of CSS file target... + "$run" mkdir -p "$css_dest_dir" # ...and create it if it doesn't exist. + "$run" sassc "${sassc_options[@]}" "$scss_src" "$css_dest" # Compile the SCSS. + + # If this is an LTR (left-to-right) SCSS source file... + if [[ "$scss_src" != *-rtl.scss ]] ; then + + # then determine what the name of the RTL source and target would be... + rtl="$scss_src_relative" # (Start with the SCSS-root-relative source path, + rtl="${rtl%-ltr.scss}" # then strip any -ltr.scss suffix, + rtl="${rtl%.scss}" # as well as any strip any .scss suffix. + rtl="${rtl}-rtl" # then finally append -rtl) + rtl_scss_src="$scss_src_root/$rtl.scss" + rtl_css_dest="$css_dest_root/$rtl.css" + + # and if the source RTL SCSS doesn't exist... + if [[ ! -f "$rtl_scss_src" ]] ; then + + # then we know that the target RTL CSS will not be generated via SCSS compilation, + # so we must auto-generate it here from the LTR CSS. + "$run" rtlcss "$css_dest" "$rtl_css_dest" + fi + fi + done < <(find "$scss_src_root" -type f -name '*.scss' \! -name '_*' -print0) +} + +# Variables, controlled by command-line options. + +subcommand="" +env="prod" + +static_root_lms="$DEFAULT_STATIC_ROOT" +static_root_cms="$static_root_lms/studio" +collect_django_settings="$DEFAULT_COLLECT_SETTINGS" + +theme_names=() +if [[ -n "$DEFAULT_THEMES_DIR" ]] ; then + theme_dirs=("$DEFAULT_THEMES_DIR") +else + theme_dirs=() +fi + +# In https://github.com/openedx/wg-developer-experience/issues/150, +# we will allow this to be configured via a new --node-modules option. +node_modules="node_modules" + +# "Boolean" variables, controlled by command-line options. +# Non-empty string is "True", empty string is "False". +# As a convention, we use the string "T" for "True". + +do_lms="T" +do_cms="T" + +# Arguments can take a few different forms, for better or for worse: +# * Positional arguments (not prefixed with '-') +# * Flag options, both short (-d) or long (--do-stuff) +# * Grouped short flag options (-dxy) +# * Single-value options, both short (-k val, -k=val) and long (--key val, --key=val) +# * Multi-value options, both short (-k val1 val2 val3) and long (--key val1 val2 val3) +# To simplify the next part of the script, we simplify the args: +# * Expand grouped short flags into individual short values (-abc => -a -b -c) +# * Remove the equals sign from value options (--key=val => --key val) +simplified_args=() +while [[ "$#" -gt 0 ]] ; do + case "$1" in + --*) + long_option="${1#--}" + if [[ "$long_option" == *'='* ]] ; then + # Long option using equals sign. + # Split it into key and value as two separate args. + long_option_key="$(echo "$long_option" | cut -d '=' -f 1)" + if [[ -z "${long_option_key}" ]] ; then + # If the key is empty, something's wrong ('-=') + fail_usage "Bad option: $1" + fi + long_option_val="$(echo "$long_option" | cut -d '=' -f 2-)" + simplified_args+=("--$long_option_key" "$long_option_val") + else + # Long option with no equals sign. Good as-is. + simplified_args+=("$1") + fi + ;; + -*) + short_option="${1#-}" + if [[ "$short_option" == *'='* ]] ; then + # Short option using equals sign. + # Split it into key and value as two separate args. + short_option_key="$(echo "$short_option" | cut -d '=' -f 1)" + if [[ "${#short_option_key}" != 1 ]] ; then + # If the key isn't one character, something's wrong ('-key=val') + fail_usage "Bad option: $1" + fi + short_option_val="$(echo "$short_option" | cut -d '=' -f 2-)" + simplified_args+=("-$short_option_key" "$short_option_val") + else + # Short option(s) with no equals sign. + # Treat each character as its own short option. + for (( i=0; i<${#short_option}; i++ )); do + simplified_args+=("-${short_option:i:1}") + done + fi + ;; + + *) + # Positional argument, or value for an option. Good as-is. + simplified_args+=("$1") + ;; + esac + shift +done + +# Now that we've simplified the arguments into a consistent format, +# loop through and process them. +set -- "${simplified_args[@]}" +while [[ "$#" -gt 0 ]] ; do + case "$1" in + + -h|--help) + show_usage + exit 0 + ;; + + -d|--dry) + run="_echo_command" + log "DRY RUN: Commands will be printed but not executed!" + shift + ;; + + -e|--env) + case "$2" in + dev|prod) + env="$2" + ;; + *) + fail_usage "expected prod or dev, got: $2" + ;; + esac + env="$2" + shift 2 + ;; + + -r|--static-root) + if [[ "$#" -eq 1 ]] ; then + fail_usage "Missing value for $1" + fi + static_root_lms="$2" + static_root_cms="$2/studio" + shift 2 + ;; + + --themes) + shift + # Append args as theme dirs until we hit an option ( -* ). + while [[ "$#" -gt 0 ]] && ! [[ "$1" = -* ]]; do + theme_names+=("$1") + shift + done + ;; + + --theme-dirs) + shift + # Append args as theme dirs until we hit an option ( -* ). + theme_dirs=() + while [[ "$#" -gt 0 ]] && [[ "$1" != -* ]]; do + theme_dirs+=("$1") + shift + done + ;; + + --systems) + shift + do_lms="" + do_cms="" + # Treat args as systems until we hit an option ( -* ). + while [[ "$#" -gt 0 ]] && ! [[ "$1" = -* ]]; do + case "$1" in + lms) + do_lms="T" + ;; + cms) + do_cms="T" + ;; + *) + fail_usage "Valid systems are: lms, cms. Got: $1" + ;; + esac + shift + done + if [[ -z "$do_lms" ]] && [[ -z "$do_cms" ]] ; then + fail_usage "You must specify one or more system" + fi + ;; + + -s|--settings) + collect_django_settings="$2" + shift 2 + ;; + + build|npm|xmodule|webpack|common|themes|collect|watch-themes) + if [[ -z "$subcommand" ]] ; then + subcommand="$1" + else + fail_usage "Cannot specify a second subcommand: $1" + fi + shift + ;; + + -*) + fail_usage "Unrecognized option: $1" + ;; + *) + fail_usage "Unexpected argument: $1" + ;; + esac +done + +if [[ -z "$subcommand" ]] ; then + fail_usage "Please specify a subcommand" +fi + +if [[ "$subcommand" = build ]] || [[ "$subcommand" = npm ]] ; then + + log_section_start "Copying static assets from node_modules..." + + # Vendor destination paths for assets. + # These are not configurable yet, but that will change as part of + # https://github.com/openedx/wg-developer-experience/issues/151 + js_vendor_path="common/static/common/js/vendor" + css_vendor_path="common/static/common/css/vendor" + edx_ui_toolkit_vendor_path="common/static/edx-ui-toolkit" + + log "Ensuring vendor directories exist..." + "$run" mkdir -p "$js_vendor_path" + "$run" mkdir -p "$css_vendor_path" + "$run" mkdir -p "$edx_ui_toolkit_vendor_path" + + log "Copying studio-frontend JS & CSS from node_modules into vendor directores..." + find "$node_modules/@edx/studio-frontend/dist" -type f -print0 | \ + while read -r -d $'\0' src_file ; do + if [[ "$src_file" = *.css ]] || [[ "$src_file" = *.css.map ]] ; then + "$run" cp --force "$src_file" "$css_vendor_path" + else + "$run" cp --force "$src_file" "$js_vendor_path" + fi + done + + log "Copying certain JS modules from node_modules into vendor directory..." + js_vendor_modules=( + "$node_modules/backbone.paginator/lib/backbone.paginator.js" + "$node_modules/backbone/backbone.js" + "$node_modules/bootstrap/dist/js/bootstrap.bundle.js" + "$node_modules/hls.js/dist/hls.js" + "$node_modules/jquery-migrate/dist/jquery-migrate.js" + "$node_modules/jquery.scrollto/jquery.scrollTo.js" + "$node_modules/jquery/dist/jquery.js" + "$node_modules/moment-timezone/builds/moment-timezone-with-data.js" + "$node_modules/moment/min/moment-with-locales.js" + "$node_modules/picturefill/dist/picturefill.js" + "$node_modules/requirejs/require.js" + "$node_modules/underscore.string/dist/underscore.string.js" + "$node_modules/underscore/underscore.js" + "$node_modules/which-country/index.js" + ) + for js_vendor_module in "${js_vendor_modules[@]}" ; do + "$run" cp --force "$js_vendor_module" "$js_vendor_path" + done + + log "Copying certain JS developer modules into vendor directory..." + if [[ "$env" = dev ]] ; then + "$run" cp --force "$node_modules/sinon/pkg/sinon.js" "$js_vendor_path" + "$run" cp --force "$node_modules/squirejs/src/Squire.js" "$js_vendor_path" + else + # TODO: https://github.com/openedx/edx-platform/issues/31768 + # In the old implementation of this scipt (pavelib/assets.py), these two + # developer libraries were copied into the JS vendor directory whether not + # the build was for prod or dev. In order to exactly match the output of + # the old script, this script will also copy them in for prod builds. + # However, in the future, it would be good to only copy them for dev + # builds. Furthermore, these libraries should not be `npm install`ed + # into prod builds in the first place. + "$run" cp --force "$node_modules/sinon/pkg/sinon.js" "$js_vendor_path" || true # "|| true" means "tolerate errors"; in this case, + "$run" cp --force "$node_modules/squirejs/src/Squire.js" "$js_vendor_path" || true # that's "tolerate if these files don't exist." + fi + + log_section_end "Done copying static assets from node_modules." +fi + +if [[ "$subcommand" = build ]] || [[ "$subcommand" = xmodule ]] ; then + + log_section_start "Processing assets from xmodule..." + + # Note: + # Collecting xmodule_assets is incompatible with setting the django path because + # of an unfortunate call to settings.configure(), so we must clear + # DJANGO_SETTINGS_MODULE before calling the script. + + "$run" env \ + "DJANGO_SETTINGS_MODULE=" \ + xmodule_assets common/static/xmodule + + log_section_end "Done processing assets from xmodule." +fi + +if [[ "$subcommand" = build ]] || [[ "$subcommand" = webpack ]] ; then + + log_section_start "Running webpack..." + + node_env="production" + if [[ "$env" = dev ]]; then + node_env="development" + fi + + "$run" env \ + "NODE_ENV=$node_env" \ + "STATIC_ROOT_LMS=$static_root_lms" \ + "STATIC_ROOT_CMS=$static_root_cms" \ + webpack --progress "--config=webpack.$env.config.js" + + log_section_end "Done running webpack." +fi + +# SCSS source roots. +lms_scss="lms/static/sass" +cms_scss="cms/static/sass" +certs_scss="lms/static/certificates/sass" + +# SCSS dependency roots (include lists are order-sensitive!) +lms_partials="lms/static/sass/partials" +cms_partials="cms/static/sass/partials" +common_includes=( + "common/static" + "common/static/sass" + "$node_modules" + "$node_modules/@edx" +) +lms_includes=( + "${common_includes[@]}" + "$lms_partials" + "$lms_scss" +) +cms_includes=( + "${common_includes[@]}" + "$lms_partials" + "$cms_partials" + "$cms_scss" +) +certs_includes=("${lms_includes[@]}") + +# CSS target roots. +lms_css="lms/static/css" +cms_css="cms/static/css" +certs_css="lms/static/certificates/css" + + +if [[ "$subcommand" = build ]] || [[ "$subcommand" = common ]] ; then + + log_section_start "Compiling static assets for common theme..." + + if [[ -n "$do_lms" ]] ; then + log "Compiling default LMS SCSS." + compile_scss_dir "$env" "$lms_scss" "$lms_css" "${lms_includes[@]}" + log "Compiling default certificates SCSS." + compile_scss_dir "$env" "$certs_scss" "$certs_css" "${certs_includes[@]}" + fi + if [[ -n "$do_cms" ]] ; then + log "Compiling default CMS SCSS." + compile_scss_dir "$env" "$cms_scss" "$cms_css" "${cms_includes[@]}" + fi + + log_section_end "Done compiling static assets for common theme." +fi + +theme_paths=() +for theme_dir in "${theme_dirs[@]}" ; do + for theme_dir_item in "$theme_dir"/* ; do + if [[ ! -d "$theme_dir_item" ]] ; then + continue + fi + if [[ "${#theme_names[@]}" = 0 ]] ; then + theme_paths+=("$theme_dir_item") + continue + fi + for theme_name in "${theme_names[@]}" ; do + if [[ "$(basename "$theme_dir_item")" = "$theme_name" ]] ; then + theme_paths+=("$theme_dir_item") + break + fi + done + done +done + +if [[ "$subcommand" = build ]] || [[ "$subcommand" = themes ]] ; then + + for theme_path in "${theme_paths[@]}" ; do + + log_section_start "Compiling static assets for custom theme: $theme_path..." + + # Theme SCSS source roots. + theme_lms_scss="$theme_path/lms/static/sass" + theme_cms_scss="$theme_path/cms/static/sass" + theme_certs_scss="$theme_path/lms/static/certificates/sass" + + # Theme SCSS dependency roots (include lists are order-sensitive!) + theme_cms_partials="$theme_path/cms/static/sass/partials" + theme_lms_partials="$theme_path/lms/static/sass/partials" + theme_lms_includes=( + "${common_includes[@]}" + "$theme_lms_partials" + "$lms_partials" + "$lms_scss" + ) + theme_cms_includes=( + "${common_includes[@]}" + "$lms_partials" + "$theme_cms_partials" + "$cms_partials" + "$cms_scss" + ) + theme_certs_includes=( + "${common_includes[@]}" + "$theme_lms_partials" + "$theme_lms_scss" + ) + + # Theme CSS target roots. + theme_lms_css="$theme_path/lms/static/css" + theme_cms_css="$theme_path/cms/static/css" + theme_certs_css="$theme_path/lms/static/certificates/css" + + if [[ -n "$do_lms" ]] ; then + if [[ -d "$theme_lms_scss" ]] ; then + log "Compiling default LMS SCSS into theme's CSS directory." + compile_scss_dir "$env" "$lms_scss" "$theme_lms_css" "${theme_lms_includes[@]}" + log "Compiling theme's LMS SCSS into theme's CSS directory." + compile_scss_dir "$env" "$theme_lms_scss" "$theme_lms_css" "${theme_lms_includes[@]}" + else + log "Theme has no LMS SCSS; skipping." + fi + if [[ -d "$theme_certs_scss" ]] ; then + log "Compiling theme's certificate SCSS into theme's CSS directory." + compile_scss_dir "$env" "$theme_certs_scss" "$theme_certs_css" "${theme_certs_includes[@]}" + else + log "Theme has no certificate SCSS; skipping." + fi + fi + if [[ -n "$do_cms" ]] ; then + if [[ -d "$theme_cms_scss" ]] ; then + log "Compiling default CMS SCSS into theme's CSS directory." + compile_scss_dir "$env" "$cms_scss" "$theme_cms_css" "${theme_cms_includes[@]}" + log "Compiling theme's CMS SCSS into theme's CSS directory." + compile_scss_dir "$env" "$theme_cms_scss" "$theme_cms_css" "${theme_cms_includes[@]}" + else + log "Theme has no CMS SCSS; skipping." + fi + fi + + log_section_end "Done compiling theme: $theme_path" + done + +fi + +if [[ "$subcommand" = collect ]] ; then + + log_section_start "Collecting static assets to be served by webserver..." + + if [[ -n "$do_lms" ]] ; then + "$run" ./manage.py lms collectstatic --noinput \ + --settings "$collect_django_settings" \ + --ignore 'fixtures' \ + --ignore 'karma_*.js' \ + --ignore 'spec' \ + --ignore 'spec_helpers' \ + --ignore 'spec-helpers' \ + --ignore 'xmodule_js' \ + --ignore 'geoip' \ + --ignore 'sass' + fi + if [[ -n "$do_cms" ]] ; then + "$run" ./manage.py cms collectstatic --noinput \ + --settings "$collect_django_settings" \ + --ignore 'fixtures' \ + --ignore 'karma_*.js' \ + --ignore 'spec' \ + --ignore 'spec_helpers' \ + --ignore 'spec-helpers' \ + --ignore 'xmodule_js' \ + --ignore 'geoip' \ + --ignore 'sass' + fi + + log_section_end "Done collecting static assets to be served by webserver." +fi + +if [[ "$subcommand" = watch-themes ]] ; then + + # TODO: implement watchers, both for themes and for other assets + fail "Watchers do not work yet." + + log_section_start "Starting watchers for theme assets..." + cleanup="echo 'Cleaning up watchers...'" + + for theme_path in "${theme_paths[@]}" ; do + theme_dir="$(dirname "$theme_path")" + theme_name="$(basename "$theme_path")" + for system in lms cms ; do + watch_path="$theme_path/$system" + watchman "--logfile=$(realpath ..)/watchman.log" watch "$watch_path" + watchman -- trigger "$watch_path" "recompile-scss-on-change--$watch_path" '*scss' -- \ + "$THIS_SCRIPT" themes \ + --theme-dirs "$theme_dir" \ + --themes "$theme_name" \ + --systems "$system" + cleanup="$cleanup ; watchman watch-del '$watch_path'" + done + done + + cleanup="$cleanup ; watchman shutdown-server ; echo 'Done cleaning up watchers.'" + # Shellcheck will warn us that "$cleanup" will be evaulated now, not + # when the signal is trapped. In fact, that's exactly what we want. + # shellcheck disable=SC2064 + trap "$cleanup" EXIT INT HUP TERM + + log_section_end "Watchers started. Use Ctrl-c to exit." + while true ; do + sleep 1 + done +fi