From e82376bb9ab7749938b076aa4bdd454118dc2f52 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 20 Apr 2023 16:01:27 -0400 Subject: [PATCH 1/5] temp: delete all checks except static assets & shellcheck --- .../add-depr-ticket-to-depr-board.yml | 19 -- .../workflows/add-remove-label-on-comment.yml | 20 -- .github/workflows/check-for-tutorial-prs.yml | 35 --- .github/workflows/ci-static-analysis.yml | 43 --- .github/workflows/commitlint.yml | 10 - .../workflows/docker-compose.yml.mysqldbdump | 23 -- .github/workflows/docker-publish.yml | 22 -- .github/workflows/docs-build-check.yml | 50 ---- .github/workflows/init/01.sql | 3 - .github/workflows/js-tests.yml | 85 ------ .github/workflows/lint-imports.yml | 52 ---- .github/workflows/lockfileversion-check.yml | 13 - .github/workflows/migrations-check-mysql8.yml | 76 ----- .github/workflows/migrations-check.yml | 93 ------ .../workflows/pr-automerge-open-release.yml | 24 -- .github/workflows/publish-ci-docker-image.yml | 35 --- .github/workflows/pylint-checks.yml | 84 ------ .github/workflows/quality-checks.yml | 82 ------ .github/workflows/self-assign-issue.yml | 12 - .github/workflows/unit-test-shards.json | 278 ------------------ .github/workflows/unit-tests-gh-hosted.yml | 121 -------- .github/workflows/unit-tests.yml | 153 ---------- .../workflows/upgrade-python-requirements.yml | 25 -- .github/workflows/verify-dunder-init.yml | 26 -- .../workflows/verify-gha-unit-tests-count.yml | 23 -- 25 files changed, 1407 deletions(-) delete mode 100644 .github/workflows/add-depr-ticket-to-depr-board.yml delete mode 100644 .github/workflows/add-remove-label-on-comment.yml delete mode 100644 .github/workflows/check-for-tutorial-prs.yml delete mode 100644 .github/workflows/ci-static-analysis.yml delete mode 100644 .github/workflows/commitlint.yml delete mode 100644 .github/workflows/docker-compose.yml.mysqldbdump delete mode 100644 .github/workflows/docker-publish.yml delete mode 100644 .github/workflows/docs-build-check.yml delete mode 100644 .github/workflows/init/01.sql delete mode 100644 .github/workflows/js-tests.yml delete mode 100644 .github/workflows/lint-imports.yml delete mode 100644 .github/workflows/lockfileversion-check.yml delete mode 100644 .github/workflows/migrations-check-mysql8.yml delete mode 100644 .github/workflows/migrations-check.yml delete mode 100644 .github/workflows/pr-automerge-open-release.yml delete mode 100644 .github/workflows/publish-ci-docker-image.yml delete mode 100644 .github/workflows/pylint-checks.yml delete mode 100644 .github/workflows/quality-checks.yml delete mode 100644 .github/workflows/self-assign-issue.yml delete mode 100644 .github/workflows/unit-test-shards.json delete mode 100644 .github/workflows/unit-tests-gh-hosted.yml delete mode 100644 .github/workflows/unit-tests.yml delete mode 100644 .github/workflows/upgrade-python-requirements.yml delete mode 100644 .github/workflows/verify-dunder-init.yml delete mode 100644 .github/workflows/verify-gha-unit-tests-count.yml diff --git a/.github/workflows/add-depr-ticket-to-depr-board.yml b/.github/workflows/add-depr-ticket-to-depr-board.yml deleted file mode 100644 index 250e394abc11..000000000000 --- a/.github/workflows/add-depr-ticket-to-depr-board.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Run the workflow that adds new tickets that are either: -# - labelled "DEPR" -# - title starts with "[DEPR]" -# - body starts with "Proposal Date" (this is the first template field) -# to the org-wide DEPR project board - -name: Add newly created DEPR issues to the DEPR project board - -on: - issues: - types: [opened] - -jobs: - routeissue: - uses: openedx/.github/.github/workflows/add-depr-ticket-to-depr-board.yml@master - secrets: - GITHUB_APP_ID: ${{ secrets.GRAPHQL_AUTH_APP_ID }} - GITHUB_APP_PRIVATE_KEY: ${{ secrets.GRAPHQL_AUTH_APP_PEM }} - SLACK_BOT_TOKEN: ${{ secrets.SLACK_ISSUE_BOT_TOKEN }} diff --git a/.github/workflows/add-remove-label-on-comment.yml b/.github/workflows/add-remove-label-on-comment.yml deleted file mode 100644 index 0f369db7d293..000000000000 --- a/.github/workflows/add-remove-label-on-comment.yml +++ /dev/null @@ -1,20 +0,0 @@ -# This workflow runs when a comment is made on the ticket -# If the comment starts with "label: " it tries to apply -# the label indicated in rest of comment. -# If the comment starts with "remove label: ", it tries -# to remove the indicated label. -# Note: Labels are allowed to have spaces and this script does -# not parse spaces (as often a space is legitimate), so the command -# "label: really long lots of words label" will apply the -# label "really long lots of words label" - -name: Allows for the adding and removing of labels via comment - -on: - issue_comment: - types: [created] - -jobs: - add_remove_labels: - uses: openedx/.github/.github/workflows/add-remove-label-on-comment.yml@master - diff --git a/.github/workflows/check-for-tutorial-prs.yml b/.github/workflows/check-for-tutorial-prs.yml deleted file mode 100644 index 6920542ac187..000000000000 --- a/.github/workflows/check-for-tutorial-prs.yml +++ /dev/null @@ -1,35 +0,0 @@ -# This workflow detects PRs that make changes to lms/templates/dashboard.html -# and only lms/templates/dashboard.html. This is the file that users are -# guided through changing in the Open edX tutorial: -# https://docs.openedx.org/en/latest/developers/quickstarts/first_openedx_pr.html#exercise-update-the-learner-dashboard - -# If this is the only file changed in the PR, we comment on the PR congratulating -# the user and letting others know that this is not a community PR in need of -# review. CODEOWNERS will tag a triaging team to provide reviews & ultimately -# close the PR. - -name: Check for Tutorial PR -description: Welcome contributors making their first PR from the tutorial -on: - pull_request: - types: [opened] - paths: - - 'lms/templates/dashboard.html' - -jobs: - # Provide helpful bot comment - comment: - runs-on: ubuntu-latest - name: provide helpful bot comment - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Comment PR - uses: thollander/actions-comment-pull-request@v1 - with: - message: | - Thank you for your pull request! Congratulations on completing the Open edX tutorial! A team member will be by to take a look shortly. - To those watching community pull requests: No need to worry about this one, a tCRIL team member will be taking care of it. - For this PR's author: If this is a PR that is NOT coming from the Open edX tutorial, please comment and let us know to disregard this message. - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml deleted file mode 100644 index 94b6b30600a8..000000000000 --- a/.github/workflows/ci-static-analysis.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Static analysis - -on: pull_request - -jobs: - tests: - name: Static analysis - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: ['3.8'] - os: ['ubuntu-20.04'] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system requirements - run: sudo apt update && sudo apt install -y libxmlsec1-dev - - - name: Install pip - run: python -m pip install -r requirements/pip.txt - - - name: Get pip cache dir - id: pip-cache-dir - run: echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install python dependencies - run: make dev-requirements - - - name: Static code analysis - run: make check-types diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml deleted file mode 100644 index fec11d6c259b..000000000000 --- a/.github/workflows/commitlint.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Run commitlint on the commit messages in a pull request. - -name: Lint Commit Messages - -on: - - pull_request - -jobs: - commitlint: - uses: openedx/.github/.github/workflows/commitlint.yml@master diff --git a/.github/workflows/docker-compose.yml.mysqldbdump b/.github/workflows/docker-compose.yml.mysqldbdump deleted file mode 100644 index 0853d250ff40..000000000000 --- a/.github/workflows/docker-compose.yml.mysqldbdump +++ /dev/null @@ -1,23 +0,0 @@ -version: '3' -services: - mysql: - image: mysql:5.7 - container_name: edx.devstack.mysql57 - ports: - - '3306:3306' - environment: - MYSQL_ROOT_PASSWORD: "" - MYSQL_ALLOW_EMPTY_PASSWORD: "yes" - volumes: - - ./init:/docker-entrypoint-initdb.d - healthcheck: - test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] - timeout: 20s - retries: 10 - edxapp: - image: edxops/edxapp:latest - command: bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' - volumes: - - ../../:/edx/app/edxapp/edx-platform - depends_on: - - mysql diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml deleted file mode 100644 index e7d9e0c1fa8e..000000000000 --- a/.github/workflows/docker-publish.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Push Docker Images - -on: - push: - branches: - - master -jobs: - # Push image to GitHub Packages. - # See also https://docs.docker.com/docker-hub/builds/ - push: - runs-on: ubuntu-latest - if: github.event_name == 'push' - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Build and Push docker image - env: - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - run : make docker_push diff --git a/.github/workflows/docs-build-check.yml b/.github/workflows/docs-build-check.yml deleted file mode 100644 index 10dd000ce0e2..000000000000 --- a/.github/workflows/docs-build-check.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Docs build - -on: - pull_request: - push: - branches: - - master - -jobs: - tests: - name: Docs build - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: ['3.8'] - os: ['ubuntu-20.04'] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system requirements - run: sudo apt update && sudo apt install -y libxmlsec1-dev - - - name: Install pip - run: python -m pip install -r requirements/pip.txt - - - name: Get pip cache dir - id: pip-cache-dir - run: echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install python dependencies - run: make dev-requirements - - - name: Install docs requirements - run: pip install -r requirements/edx/doc.txt - - - name: Docs build - run: make docs diff --git a/.github/workflows/init/01.sql b/.github/workflows/init/01.sql deleted file mode 100644 index 93d3a107e35e..000000000000 --- a/.github/workflows/init/01.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE DATABASE IF NOT EXISTS `edxapp`; -CREATE DATABASE IF NOT EXISTS `edxapp_csmh`; -GRANT ALL PRIVILEGES ON *.* TO 'edxapp001'@'%' IDENTIFIED BY 'password'; diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml deleted file mode 100644 index 743b5286d87e..000000000000 --- a/.github/workflows/js-tests.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: Javascript tests - -on: - pull_request: - branches: - - master - push: - branches: - - master - -jobs: - run_tests: - name: JS - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ ubuntu-20.04 ] - node-version: [ 16 ] - python-version: [ 3.8 ] - - steps: - - - uses: actions/checkout@v2 - - name: Fetch master to compare coverage - run: git fetch --depth=1 origin master - - - name: Setup Node - uses: actions/setup-node@v2 - with: - node-version: ${{ matrix.node-version }} - - - name: Setup npm - run: npm i -g npm@8.5.x - - - name: Install Firefox 61.0 - run: | - sudo apt-get purge firefox - wget "https://ftp.mozilla.org/pub/firefox/releases/61.0/linux-x86_64/en-US/firefox-61.0.tar.bz2" - tar -xjf firefox-61.0.tar.bz2 - sudo mv firefox /opt/firefox - sudo ln -s /opt/firefox/firefox /usr/bin/firefox - - - name: Install Required System Packages - run: sudo apt-get update && sudo apt-get install libxmlsec1-dev ubuntu-restricted-extras xvfb - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/base.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install Required Python Dependencies - run: | - make base-requirements - - - uses: c-hive/gha-npm-cache@v1 - - name: Run JS Tests - env: - TEST_SUITE: js-unit - SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh - run: | - npm install -g jest - xvfb-run --auto-servernum ./scripts/all-tests.sh - - - name: Save Job Artifacts - uses: actions/upload-artifact@v2 - with: - name: Build-Artifacts - path: | - reports/**/* - test_root/log/*.png - test_root/log/*.log - **/TEST-*.xml diff --git a/.github/workflows/lint-imports.yml b/.github/workflows/lint-imports.yml deleted file mode 100644 index 63caae452f3a..000000000000 --- a/.github/workflows/lint-imports.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Lint Python Imports - -on: - pull_request: - push: - branches: - - master - -jobs: - - lint-imports: - name: Lint Python Imports - runs-on: ubuntu-20.04 - - steps: - - name: Check out branch - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - - name: Install system requirements - run: sudo apt update && sudo apt install -y libxmlsec1-dev - - - name: Install pip - run: python -m pip install -r requirements/pip.txt - - - name: Get pip cache dir - id: pip-cache-dir - run: echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install python dependencies - run: pip install -r requirements/edx/development.txt - - # As long there are sub-projects[1] in edx-platform, we analyze each - # project separately here, in order to make import-linting errors easier - # to pinpoint. - # - # [1] https://openedx.atlassian.net/browse/BOM-2579 - - - name: Analyze imports (repo root) - run: make lint-imports diff --git a/.github/workflows/lockfileversion-check.yml b/.github/workflows/lockfileversion-check.yml deleted file mode 100644 index 42312e8cbf2d..000000000000 --- a/.github/workflows/lockfileversion-check.yml +++ /dev/null @@ -1,13 +0,0 @@ -#check package-lock file version - -name: Lockfile Version check - -on: - push: - branches: - - master - pull_request: - -jobs: - version-check: - uses: openedx/.github/.github/workflows/lockfileversion-check.yml@master diff --git a/.github/workflows/migrations-check-mysql8.yml b/.github/workflows/migrations-check-mysql8.yml deleted file mode 100644 index 627bceb9ba9e..000000000000 --- a/.github/workflows/migrations-check-mysql8.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Migrations check on MySql 8.0 - -on: - workflow_dispatch: - pull_request: - push: - branches: - - master - -jobs: - check_migrations: - name: check migrations mysql8 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ ubuntu-20.04 ] - python-version: [ 3.8 ] - - steps: - - name: Checkout repo - uses: actions/checkout@v2 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system Packages - run: | - sudo apt-get update - sudo apt-get install -y libxmlsec1-dev - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Ubuntu and sql Versions - run: | - lsb_release -a - mysql -V - - - name: Install Python dependencies - run: | - make dev-requirements - - - name: Initiate Services - run: | - sudo systemctl start mongod - sudo /etc/init.d/mysql start - - - name: Reset mysql password - run: | - cat < Date: Wed, 19 Apr 2023 09:12:11 -0400 Subject: [PATCH 2/5] build: rewrite static asset build TODO description Closes https://github.com/openedx/edx-platform/issues/31604 --- .../0017-reimplement-asset-processing.rst | 8 +- scripts/build-assets.sh | 620 ++++++++++++++++++ uwsgi.ini | 0 3 files changed, 624 insertions(+), 4 deletions(-) create mode 100755 scripts/build-assets.sh create mode 100755 uwsgi.ini diff --git a/docs/decisions/0017-reimplement-asset-processing.rst b/docs/decisions/0017-reimplement-asset-processing.rst index 7f16dcc249ff..c041a7c2e5e9 100644 --- a/docs/decisions/0017-reimplement-asset-processing.rst +++ b/docs/decisions/0017-reimplement-asset-processing.rst @@ -167,7 +167,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* Bash wrapper around a call to webpack. The script will accept parameters rather than looking up Django settings itself. - The print_setting command will still be available for distributions to use to extract ``STATIC_ROOT`` from Django settings, but it will only need to be run once. As described in **Build Configuration** below, unnecessary Django settings will be removed. Some distributions may not even need to look up ``STATIC_ROOT``; Tutor, for example, will probably render ``STATIC_ROOT`` directly into the environment variable ``OPENEDX_BUILD_ASSETS_OPTS`` variable, described in the **Build Configuration**. + The print_setting command will still be available for distributions to use to extract ``STATIC_ROOT`` from Django settings, but it will only need to be run once. As described in **Build Configuration** below, unnecessary Django settings will be removed. Some distributions may not even need to look up ``STATIC_ROOT``; Tutor, for example, will probably render ``STATIC_ROOT`` directly into the environment variable ``EDX_PLATFORM_BUILD_ASSETS_OPTS`` variable, described in the **Build Configuration**. * - + **Build stage 4: Compile default SCSS** into CSS for legacy LMS/CMS frontends. @@ -228,12 +228,12 @@ The three top-level edx-platform asset processing actions are *build*, *collect* Build Configuration ------------------- -``scripts/build-assets.sh`` will accept various command-line options to configure the build. It will also accept the same options in the form of the ``OPENEDX_BUILD_ASSETS_OPTS`` enviroment variable. Options from the environment variable will be processed first, and then overridden by options provided on the command line. The environment variable allows distributions like Tutor to seed the build script with "defaults" in the event that the upstream defaults are not sufficient, while still allowing individual operators to run the script with whichever options they like. +``scripts/build-assets.sh`` will accept various command-line options to configure the build. It will also accept the same options in the form of the ``EDX_PLATFORM_BUILD_ASSETS_OPTS`` enviroment variable. Options from the environment variable will be processed first, and then overridden by options provided on the command line. The environment variable allows distributions like Tutor to seed the build script with "defaults" in the event that the upstream defaults are not sufficient, while still allowing individual operators to run the script with whichever options they like. As a concrete example, the default value of ``--theme-dirs`` will be ``''`` (that is: no themes) and the default value of ``--static-root`` will be ``./test_root/static``. Neither of those are suitable for Tutor. Instead, Tutor will set the environment variable in its Dockerfile:: ... - ENV OPENEDX_BUILD_ASSETS_OPTS '--theme-dirs /openedx/themes --static-root /openedx/staticfiles' + ENV EDX_PLATFORM_BUILD_ASSETS_OPTS '--theme-dirs /openedx/themes --static-root /openedx/staticfiles' ... Later, in the container, a user might run:: @@ -261,7 +261,7 @@ Furthermore, to facilitate a Python-free build reimplementation, we will remove - Set an environment variable before calling build-assets.sh:: - OPENEDX_BUILD_ASSETS_OPTS=\ + EDX_PLATFORM_BUILD_ASSETS_OPTS=\ '--webpack-config path/to/webpack.my.config.js' * - JS_ENV_EXTRA_CONFIG diff --git a/scripts/build-assets.sh b/scripts/build-assets.sh new file mode 100755 index 000000000000..c20393add0c1 --- /dev/null +++ b/scripts/build-assets.sh @@ -0,0 +1,620 @@ +#!/usr/bin/env bash +ABOUT="Build static assets for edx-platform." + +# Enable stricter error handling. +set -euo pipefail + + +######################################################################### +# ENVIRONMENT VARIABLES +######################################################################### + +# Actual path to this script +THIS_SCRIPT="$0" + +# Script name for logging & errors. Default to actual path. +SCRIPT_NAME="${SCRIPT_NAME:-$THIS_SCRIPT}" + +# Default command line options. +# These are processed before any other command line options, so setting +# this environment variable is a way of setting custom 'default' values +# for this script. These defaults even show up in show_usage, because +# they are processed before --help is processed +# LIMITATION: Paths in arguments may not contain spaces. Quoting +# will not work. +DEFAULT_OPTS="${EDX_PLATFORM_BUILD_ASSETS_OPTS:-}" + +# Extra fields to inject into the `process.env` of the Webpack build. +# Should be a valid JSON string representing an object. +# Defaults to empty object ('{}'). +export JS_ENV_EXTRA_CONFIG="${JS_ENV_EXTRA_CONFIG:-{}}" + + +######################################################################### +# CONSTANTS +# Some of these are lowercase because they may become variables one day. +######################################################################### + +# Codes for colored terminal output. +COL_LOG="\e[36m" # Log/step/section color (cyan) +COL_RUN="\e[35m" # Executed code echo color (purple) +COL_ERR="\e[31m" # Error color (red) +COL_OFF="\e[0m" # Normal color + +# Input directories +node_modules="node_modules" +lms_scss="lms/static/sass" +cms_scss="cms/static/sass" +certs_scss="lms/static/certificates/sass" +lms_partials="lms/static/sass/partials" +cms_partials="cms/static/sass/partials" +common_includes=( + "common/static" + "common/static/sass" + "$node_modules" + "$node_modules/@edx" +) +lms_includes=( + "${common_includes[@]}" + "$lms_partials" + "$lms_scss" +) +cms_includes=( + "${common_includes[@]}" + "$lms_partials" + "$cms_partials" + "$cms_scss" +) +certs_includes=("${lms_includes[@]}") + +# Destination directories for generated assets +vendor_js="common/static/common/js/vendor" +vendor_css="common/static/common/css/vendor" +xmodule_fragments="common/static/xmodule" +lms_css="lms/static/css" +cms_css="cms/static/css" +certs_css="lms/static/certificates/css" + + +######################################################################### +# GLOBAL VARIABLES +# These get modified when we process command-line arguments. +######################################################################### + +# What to build +stage="" +env="prod" +systems=("lms" "cms") + +# Destination directories for generated assets +static_root="test_root/static" + +# Themes: for each (theme_dir, theme_name) pair, +# theme_dir/theme_name is a potential theme path that +# we'll check for. +theme_dirs=() # Empty is treated as 'none' +theme_names=() # Empty is treated as 'all in search dirs' + +# run: a 'function pointer' to either _echo_and_run_command or _echo_command, +# both of which are defined below. +# +# All shell commands invoked by this script that mutate the system +# are wrapped in a call to '"$run" ...'. This gives us two benefits: +# 1. Commands are always echoed before they are executed, making it +# easier for users to understand & debug the script from its output. +# 2. If the user passses --dry-run, we set run="_echo_command", +# allowing to the script to be run with command printed but not executed. +run="_echo_and_run_command" + + +######################################################################### +# FUNCTIONS +######################################################################### + +# Print script usage information. +show_usage ( ) { + echo "Usage: $SCRIPT_NAME [] [] []" + echo + echo "$ABOUT" + echo + echo "You can specify one build stage." + echo "Otherwise, all of them will be run." + echo + echo "Stages:" + echo " npm Copy npm-installed assets" + echo " xmodule Copy XModule fragments" + echo " webpack Run Webpack" + echo " css Compile default SCSS" + echo " themes Compile themes' SCSS" + echo + echo "Options:" + echo " -h|--help Display this help message." + echo " -d|--dry-run Print shell commands but do not run them." + echo " -e|--env Compilation environment (prod or dev)." + echo " Default: $env" + echo " -r|--static-root Path for Webpack output." + echo " Default: $static_root" + echo " --systems ... Specify lms and/or cms." + echo " Default: ${systems[*]:-(none)}" + echo " --theme-dirs ... Specify one or more theme search dirs." + echo " Default: ${theme_dirs[*]:-(none)}" + echo " --themes ... Themes to compile from theme-dirs." + echo " Default: ${theme_names[*]:-(all)}" +} + +# Print a formatted error message. +show_error ( ) { + local error_message="$1" + + echo -e "${COL_ERR}${SCRIPT_NAME}: error: ${error_message}${COL_OFF}" +} + +# Print a formatted error message, and exit the script unsuccessfully. +fail ( ) { + local error_message="$1" + + show_error "$error_message" + exit 1 +} + +# Print a formatted error message, tell the user how to view the script's usage info, +# and exit the script unsucessfully. Use this function when the error is simply +# that the script has been called wrong. +fail_usage ( ) { + local error_message="$1" + + show_error "$error_message" + echo + echo "Try '$SCRIPT_NAME -h' or '$SCRIPT_NAME --help' for more information." + exit 1 +} + +# Log the beginning of a "section" (a larger part of the script). +log_section_start ( ) { + local section_description="$*" + + echo -e "${COL_LOG}=====================================================================================$COL_OFF" + echo -e "${COL_LOG} $section_description $COL_OFF" + echo -e "${COL_LOG}-------------------------------------------------------------------------------$COL_OFF" +} + +# Log the end of a "section" (a larger part of the script). +log_section_end ( ) { + local section_description="$*" + + echo -e "${COL_LOG}-------------------------------------------------------------------------------$COL_OFF" + echo -e "${COL_LOG} $section_description $COL_OFF" + echo -e "${COL_LOG}=====================================================================================$COL_OFF" +} + +# Log a line of information. +log ( ) { + local log_line="$*" + + echo -e "${COL_LOG}$SCRIPT_NAME: $log_line $COL_OFF" +} + +# Print a shell command (without running it). +# Command should be passed as separate arguments. +_echo_command ( ) { + local command_components=("$@") + + echo -e "${COL_RUN}${command_components[*]}${COL_OFF}" +} + +# Print & execute a shell command. +# Command should be passed as separate arguments. +_echo_and_run_command ( ) { + local command_components=("$@") + + _echo_command "${command_components[@]}" + "${command_components[@]}" +} + +# Compile a directory of SCSS into CSS, generating RTL CSS as needed. +# +# TODO: Unlike its Python API, libsass-python's CLI (sassc) only supports compiling individual +# SCSS files, not entire directories. However, the CLIs for dart-sass and node-sass +# both *do* support compiling entire directories. So, if/when we upgrade to one of those +# libraries, much of this function can be replaced with a single CLI call. +# https://github.com/openedx/edx-platform/issues/31607 +compile_scss_dir ( ) { + local scss_env="$1" # 1: Environment (dev or prod). For output styling. + local scss_src_root="$2" # 2: Path to source directory containing SCSS. + local css_dest_root="$3" # 3: Path to target directory for CSS. + shift 3 + local include_paths=("$@") # Remaining args: Search paths for SCSS imports. + + local sassc_options=() + + # Add output-style option, depending on environment. + if [[ "$scss_env" == dev ]] ; then + sassc_options+=("--output-style=nested" "--sourcemap") + else + sassc_options+=("--output-style=compressed") + fi + + # For each include path, add it to the list of SCSS compile options. + for include_path in "${include_paths[@]}" ; do + sassc_options+=("--include-path=$include_path") + done + + # For each SCSS file $scss_src within $scss_src_root (recursive), + # excluding underscore-prefixed (i.e., partial) SCSS files... + while read -r -d $'\0' scss_src ; do + + # Translate source path into destination path: + scss_src_relative="${scss_src#"$scss_src_root"}" # Chop off SCSS root dir prefix. + css_dest_relative="${scss_src_relative%.scss}.css" # Replace file extension. + css_dest="$css_dest_root/$css_dest_relative" # Prepend CSS root dir. + + css_dest_dir="$(dirname "$css_dest")" # Find immediate parent dir of CSS file target... + "$run" mkdir -p "$css_dest_dir" # ...and create it if it doesn't exist. + "$run" sassc "${sassc_options[@]}" "$scss_src" "$css_dest" # Compile the SCSS. + + # If this is an LTR (left-to-right) SCSS source file... + if [[ "$scss_src" != *-rtl.scss ]] ; then + + # then determine what the name of the RTL source and target would be... + rtl="$scss_src_relative" # (Start with the SCSS-root-relative source path, + rtl="${rtl%-ltr.scss}" # then strip any -ltr.scss suffix, + rtl="${rtl%.scss}" # as well as any strip any .scss suffix. + rtl="${rtl}-rtl" # then finally append -rtl) + rtl_scss_src="$scss_src_root/$rtl.scss" + rtl_css_dest="$css_dest_root/$rtl.css" + + # and if the source RTL SCSS doesn't exist... + if [[ ! -f "$rtl_scss_src" ]] ; then + + # then we know that the target RTL CSS will not be generated via SCSS compilation, + # so we must auto-generate it here from the LTR CSS. + "$run" rtlcss "$css_dest" "$rtl_css_dest" + fi + fi + done < <(find "$scss_src_root" -type f -name '*.scss' \! -name '_*' -print0) +} + + +######################################################################### +# COMMAND-LINE ARGUMENT PROCESSING +######################################################################### + +# Stick the DEFAULT_OPTS in front of the actual command-line +# arguments ($@) so that they are processed below as default arguments. +# We split DEFAULT_OPTS on spaces, which shellcheck doesn't like +# (because it won't work on filenames with spaces) but there isn't really +# a better way to do this. +# shellcheck disable=SC2086 +set -- $DEFAULT_OPTS "$@" + +# Loop through arguments list. +while [[ "$#" -gt 0 ]] ; do + case "$1" in + + -h|--help) + show_usage + exit 0 + ;; + + -d|--dry-run) + run="_echo_command" + log "DRY RUN: Commands will be printed but not executed!" + shift + ;; + + -e|--env) + case "$2" in + dev|prod) + env="$2" + ;; + *) + fail_usage "expected prod or dev, got: $2" + ;; + esac + env="$2" + shift 2 + ;; + + -r|--static-root) + if [[ "$#" -eq 1 ]] ; then + fail_usage "Missing value for $1" + fi + static_root="$2" + shift 2 + ;; + + --themes) + shift + theme_names=() + # Append args as theme names until we hit an option ( -* ). + while [[ "$#" -gt 0 ]] && ! [[ "$1" = -* ]]; do + theme_names+=("$1") + shift + done + ;; + + --theme-dirs) + shift + theme_dirs=() + # Append args as theme dirs until we hit an option ( -* ). + while [[ "$#" -gt 0 ]] && [[ "$1" != -* ]]; do + theme_dirs+=("$1") + shift + done + ;; + + --systems) + shift + systems=() + # Treat args as systems until we hit an option ( -* ). + while [[ "$#" -gt 0 ]] && ! [[ "$1" = -* ]]; do + systems+=("$1") + shift + done + ;; + + npm|xmodule|webpack|css|themes) + if [[ -z "$stage" ]] ; then + stage="$1" + else + fail_usage "Cannot specify a second stage: $1" + fi + shift + ;; + + -*) + fail_usage "Unrecognized option: $1" + ;; + *) + fail_usage "Unexpected argument: $1" + ;; + esac +done + +# Boolean flags for each system. +# Non-empty string is true; empty string is false. +do_lms="" +do_cms="" +if [[ "${#systems[@]}" -eq 0 ]] ; then + fail_usage "You must specify one or more system" +fi +for system in "${systems[@]}" ; do + case "$system" in + lms) + do_lms="T" + ;; + cms) + do_cms="T" + ;; + *) + fail_usage "Valid systems are: lms, cms. Got: $1" + ;; + esac +done + +# Static roots for Webpack output +static_root_lms="$static_root" +static_root_cms="$static_root/studio" + +# Build a list of paths to themes to compile. +# For each theme search folder... +theme_paths=() +for theme_dir in "${theme_dirs[@]}" ; do + + # Make sure the search folder exists. + if [[ ! -d "$theme_dir" ]] ; then + fail "provided --theme-dir is not a directory: $theme_dir" + fi + + # For each item in the search folder... + for theme_dir_item in "$theme_dir"/* ; do + + # Ensure the item is a subfolder. + if [[ ! -d "$theme_dir_item" ]] ; then + continue + fi + # If it's a subfolder, then we have to assume it's a theme. + # It might not be a theme, but that's OK. Compiling a folder that's + # not a theme is basically a no-op. + + # If no theme names were provided, then that means we want every theme + # in every search dir. So, add this subfolder to our list of themes to + # compile. + if [[ "${#theme_names[@]}" = 0 ]] ; then + theme_paths+=("$theme_dir_item") + continue + fi + + # If theme names were provided, then add this subfolder if and only if + # it matches one of the provided theme names. + for theme_name in "${theme_names[@]}" ; do + if [[ "$(basename "$theme_dir_item")" = "$theme_name" ]] ; then + theme_paths+=("$theme_dir_item") + break + fi + done + done +done + + +######################################################################### +# BUILD STAGES +# We do one of these (if $stage is set) or all of them (if unset). +######################################################################### + +if [[ -z "$stage" ]] || [[ "$stage" = npm ]] ; then + + log_section_start "Copying npm-installed assets..." + + log "Ensuring vendor directories exist..." + "$run" mkdir -p "$vendor_js" + "$run" mkdir -p "$vendor_css" + + log "Copying studio-frontend JS & CSS from node_modules into vendor directores..." + while read -r -d $'\0' src_file ; do + if [[ "$src_file" = *.css ]] || [[ "$src_file" = *.css.map ]] ; then + "$run" cp --force "$src_file" "$vendor_css" + else + "$run" cp --force "$src_file" "$vendor_js" + fi + done < <(find "$node_modules/@edx/studio-frontend/dist" -type f -print0) + + log "Copying certain JS modules from node_modules into vendor directory..." + "$run" cp --force \ + "$node_modules/backbone.paginator/lib/backbone.paginator.js" \ + "$node_modules/backbone/backbone.js" \ + "$node_modules/bootstrap/dist/js/bootstrap.bundle.js" \ + "$node_modules/hls.js/dist/hls.js" \ + "$node_modules/jquery-migrate/dist/jquery-migrate.js" \ + "$node_modules/jquery.scrollto/jquery.scrollTo.js" \ + "$node_modules/jquery/dist/jquery.js" \ + "$node_modules/moment-timezone/builds/moment-timezone-with-data.js" \ + "$node_modules/moment/min/moment-with-locales.js" \ + "$node_modules/picturefill/dist/picturefill.js" \ + "$node_modules/requirejs/require.js" \ + "$node_modules/underscore.string/dist/underscore.string.js" \ + "$node_modules/underscore/underscore.js" \ + "$node_modules/which-country/index.js" \ + "$vendor_js" + + log "Copying certain JS developer modules into vendor directory..." + if [[ "$env" = dev ]] ; then + "$run" cp --force "$node_modules/sinon/pkg/sinon.js" "$vendor_js" + "$run" cp --force "$node_modules/squirejs/src/Squire.js" "$vendor_js" + else + # TODO: https://github.com/openedx/edx-platform/issues/31768 + # In the old implementation of this scipt (pavelib/assets.py), these two + # developer libraries were copied into the JS vendor directory whether not + # the build was for prod or dev. In order to exactly match the output of + # the old script, this script will also copy them in for prod builds. + # However, in the future, it would be good to only copy them for dev + # builds. Furthermore, these libraries should not be `npm install`ed + # into prod builds in the first place. + "$run" cp --force "$node_modules/sinon/pkg/sinon.js" "$vendor_js" || true # "|| true" means "tolerate errors"; in this case, + "$run" cp --force "$node_modules/squirejs/src/Squire.js" "$vendor_js" || true # that's "tolerate if these files don't exist." + fi + + log_section_end "Done copying npm-installed assets." +fi + +if [[ -z "$stage" ]] || [[ "$stage" = xmodule ]] ; then + + log_section_start "Copying XModule fragments..." + + # Note: + # Copying xmodule_assets is incompatible with setting the django path because + # of an unfortunate call to settings.configure(), so we must clear + # DJANGO_SETTINGS_MODULE before calling the script. + + "$run" env \ + "DJANGO_SETTINGS_MODULE=" \ + xmodule_assets "$xmodule_fragments" + + log_section_end "Done copying XModule fragments." +fi + +if [[ -z "$stage" ]] || [[ "$stage" = webpack ]] ; then + + log_section_start "Running Webpack..." + + node_env="production" + if [[ "$env" = dev ]]; then + node_env="development" + fi + + "$run" env \ + "NODE_ENV=$node_env" \ + "STATIC_ROOT_LMS=$static_root_lms" \ + "STATIC_ROOT_CMS=$static_root_cms" \ + webpack --progress "--config=webpack.$env.config.js" + + log_section_end "Done running Webpack." +fi + +if [[ -z "$stage" ]] || [[ "$stage" = css ]] ; then + + log_section_start "Compiling default SCSS..." + + if [[ -n "$do_lms" ]] ; then + log "Compiling default LMS SCSS." + compile_scss_dir "$env" "$lms_scss" "$lms_css" "${lms_includes[@]}" + log "Compiling default certificates SCSS." + compile_scss_dir "$env" "$certs_scss" "$certs_css" "${certs_includes[@]}" + fi + if [[ -n "$do_cms" ]] ; then + log "Compiling default CMS SCSS." + compile_scss_dir "$env" "$cms_scss" "$cms_css" "${cms_includes[@]}" + fi + + log_section_end "Done compiling default SCSS." +fi + +if [[ -z "$stage" ]] || [[ "$stage" = themes ]] ; then + + for theme_path in "${theme_paths[@]}" ; do + + log_section_start "Compiling SCSS for theme at: $theme_path..." + + # Theme SCSS source roots. + theme_lms_scss="$theme_path/lms/static/sass" + theme_cms_scss="$theme_path/cms/static/sass" + theme_certs_scss="$theme_path/lms/static/certificates/sass" + + # Theme SCSS dependency roots (include lists are order-sensitive!) + theme_cms_partials="$theme_path/cms/static/sass/partials" + theme_lms_partials="$theme_path/lms/static/sass/partials" + theme_lms_includes=( + "${common_includes[@]}" + "$theme_lms_partials" + "$lms_partials" + "$lms_scss" + ) + theme_cms_includes=( + "${common_includes[@]}" + "$lms_partials" + "$theme_cms_partials" + "$cms_partials" + "$cms_scss" + ) + theme_certs_includes=( + "${common_includes[@]}" + "$theme_lms_partials" + "$theme_lms_scss" + ) + + # Theme CSS target roots. + theme_lms_css="$theme_path/lms/static/css" + theme_cms_css="$theme_path/cms/static/css" + theme_certs_css="$theme_path/lms/static/certificates/css" + + if [[ -n "$do_lms" ]] ; then + if [[ -d "$theme_lms_scss" ]] ; then + log "Compiling default LMS SCSS into theme's CSS directory." + compile_scss_dir "$env" "$lms_scss" "$theme_lms_css" "${theme_lms_includes[@]}" + log "Compiling theme's LMS SCSS into theme's CSS directory." + compile_scss_dir "$env" "$theme_lms_scss" "$theme_lms_css" "${theme_lms_includes[@]}" + else + log "Theme has no LMS SCSS; skipping." + fi + if [[ -d "$theme_certs_scss" ]] ; then + log "Compiling theme's certificate SCSS into theme's CSS directory." + compile_scss_dir "$env" "$theme_certs_scss" "$theme_certs_css" "${theme_certs_includes[@]}" + else + log "Theme has no certificate SCSS; skipping." + fi + fi + if [[ -n "$do_cms" ]] ; then + if [[ -d "$theme_cms_scss" ]] ; then + log "Compiling default CMS SCSS into theme's CSS directory." + compile_scss_dir "$env" "$cms_scss" "$theme_cms_css" "${theme_cms_includes[@]}" + log "Compiling theme's CMS SCSS into theme's CSS directory." + compile_scss_dir "$env" "$theme_cms_scss" "$theme_cms_css" "${theme_cms_includes[@]}" + else + log "Theme has no CMS SCSS; skipping." + fi + fi + + log_section_end "Done compiling SCSS for theme at: $theme_path" + done + +fi diff --git a/uwsgi.ini b/uwsgi.ini new file mode 100755 index 000000000000..e69de29bb2d1 From 95ce6e52e3a0bae9b964a98fc22a2a476aa4dd9e Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Tue, 25 Apr 2023 11:34:27 -0400 Subject: [PATCH 3/5] temp: disable source comments & maps for CI purposes We want to compare the legacy and new outputs for SCSS compilation. However, there is one silly difference between the two that we can not reconcile: * The legacy build (using Python `sass.compile`) adds source comments when in dev mode, but cannot output source maps. * The new build (using CLI `sassc`) can only add source comments if it generates source maps. So, for the purposes of making our comparison pass in CI, this commit: * removes source_comments from legacy build, and * removes --sourcemap from new build. When we finally merge everything in, though, and no longer need this comparison in CI, we will just enable source maps on the new build. --- pavelib/assets.py | 3 --- scripts/build-assets.sh | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pavelib/assets.py b/pavelib/assets.py index 003c76487585..a826277ba36e 100644 --- a/pavelib/assets.py +++ b/pavelib/assets.py @@ -523,10 +523,8 @@ def _compile_sass(system, theme, debug, force, timing_info): # determine css out put style and source comments enabling if debug: - source_comments = True output_style = 'nested' else: - source_comments = False output_style = 'compressed' for dirs in sass_dirs: @@ -558,7 +556,6 @@ def _compile_sass(system, theme, debug, force, timing_info): sass.compile( dirname=(sass_source_dir, css_dir), include_paths=COMMON_LOOKUP_PATHS + lookup_paths, - source_comments=source_comments, output_style=output_style, ) diff --git a/scripts/build-assets.sh b/scripts/build-assets.sh index c20393add0c1..f47e67b4f989 100755 --- a/scripts/build-assets.sh +++ b/scripts/build-assets.sh @@ -229,7 +229,7 @@ compile_scss_dir ( ) { # Add output-style option, depending on environment. if [[ "$scss_env" == dev ]] ; then - sassc_options+=("--output-style=nested" "--sourcemap") + sassc_options+=("--output-style=nested") else sassc_options+=("--output-style=compressed") fi From b2f3f2a7884685a2c38c825ee663de16dbe46006 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 24 Apr 2023 16:50:56 -0400 Subject: [PATCH 4/5] build: static-dev makefile rule (WIP) --- Makefile | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Makefile b/Makefile index fec1c2caf2e7..e3105da78cc0 100644 --- a/Makefile +++ b/Makefile @@ -175,3 +175,13 @@ lint-imports: # Part of https://github.com/openedx/wg-developer-experience/issues/136 ubuntu-requirements: ## Install ubuntu 22.04 system packages needed for `pip install` to work on ubuntu. sudo apt install libmysqlclient-dev libxmlsec1-dev + +.PHONY: +static-dev: + LMS_CFG=lms/envs/bok_choy.yml CMS_CFG=cms/envs/bok_choy.yml DJANGO_SETTINGS_MODULE=lms.envs.devstack \ + scripts/build-assets.sh --env dev --static-root "$$(./manage.py lms print_setting STATIC_ROOT)" + LMS_CFG=lms/envs/bok_choy.yml CMS_CFG=cms/envs/bok_choy.yml DJANGO_SETTINGS_MODULE=lms.envs.devstack \ + ./manage.py lms collectstatic --noinput + LMS_CFG=lms/envs/bok_choy.yml CMS_CFG=cms/envs/bok_choy.yml DJANGO_SETTINGS_MODULE=cms.envs.devstack \ + ./manage.py cms collectstatic --noinput + From 3c8d67df053225284e60d9975a46a381b347cd19 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 20 Apr 2023 15:41:11 -0400 Subject: [PATCH 5/5] test: check new assets processing in static assets check --- .github/workflows/static-assets-check.yml | 155 +++++++++++++++++++++- 1 file changed, 149 insertions(+), 6 deletions(-) diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 6bc878102e93..97508d7b5b95 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -1,4 +1,4 @@ -name: static assets check for lms and cms +name: Static Assets on: pull_request: @@ -6,9 +6,13 @@ on: branches: - master +defaults: + run: + shell: bash + jobs: static_assets_check: - name: static-assets-check + name: Static assets check (legacy) runs-on: ${{ matrix.os }} strategy: matrix: @@ -54,7 +58,7 @@ jobs: - name: Install Required Python Dependencies run: | - make base-requirements + make dev-requirements - name: Initiate Mongo DB Service run: sudo systemctl start mongod @@ -62,11 +66,150 @@ jobs: - name: Add node_modules bin to $Path run: echo $GITHUB_WORKSPACE/node_modules/.bin >> $GITHUB_PATH - - name: Run Static Assets Check + - name: Look up Static Root + id: look-up-static-root + env: + LMS_CFG: lms/envs/bok_choy.yml + CMS_CFG: cms/envs/bok_choy.yml + DJANGO_SETTINGS_MODULE: lms.envs.devstack + run: echo "STATIC_ROOT=$(./manage.py lms print_setting STATIC_ROOT)" >> "$GITHUB_OUTPUT" + + - name: Build & Collect Static Assets env: LMS_CFG: lms/envs/bok_choy.yml CMS_CFG: cms/envs/bok_choy.yml + run: | + paver update_assets lms --debug + paver update_assets cms --debug + + - name: Generate List of Collected Staticfiles with Hashes + run: find "${{ steps.look-up-static-root.outputs.STATIC_ROOT }}" -type f | sort | xargs sha1sum > staticfiles-hashes-legacy.txt + + - name: Upload List of Collected Staticfiles with Hashes + uses: actions/upload-artifact@v3 + with: + name: staticfiles-hashes-legacy + path: staticfiles-hashes-legacy.txt + + static_assets_check_new: + name: Static assets check (new) + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-20.04 ] + python-version: [ 3.8 ] + node-version: [ 16 ] + npm-version: [ 8.5.x ] + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system Packages + run: | + sudo apt-get update + sudo apt-get install libxmlsec1-dev pkg-config + + - name: Setup Node + uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.node-version }} + + - name: Setup npm + run: npm i -g npm@${{ matrix.npm-version }} + + - name: Get pip cache dir + id: pip-cache-dir + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: Cache pip dependencies + id: cache-dependencies + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} + restore-keys: ${{ runner.os }}-pip- + - name: Install Required Python Dependencies run: | - paver update_assets lms - paver update_assets cms + make dev-requirements + + - name: Initiate Mongo DB Service + run: sudo systemctl start mongod + + - name: Add node_modules bin to $Path + run: echo $GITHUB_WORKSPACE/node_modules/.bin >> $GITHUB_PATH + + - name: Look up Static Root + id: look-up-static-root + env: + LMS_CFG: lms/envs/bok_choy.yml + CMS_CFG: cms/envs/bok_choy.yml + DJANGO_SETTINGS_MODULE: lms.envs.devstack + run: echo "STATIC_ROOT=$(./manage.py lms print_setting STATIC_ROOT)" >> "$GITHUB_OUTPUT" + + - name: Install Node Modules + run: npm clean-install + + - name: Build Static Assets + env: + LMS_CFG: lms/envs/bok_choy.yml + CMS_CFG: cms/envs/bok_choy.yml + DJANGO_SETTINGS_MODULE: lms.envs.devstack + run: scripts/build-assets.sh --env dev --static-root "${{ steps.look-up-static-root.outputs.STATIC_ROOT }}" + + - name: Collect LMS Static Assets + env: + LMS_CFG: lms/envs/bok_choy.yml + DJANGO_SETTINGS_MODULE: lms.envs.devstack + run: ./manage.py lms collectstatic --noinput + + - name: Collect CMS Static Assets + env: + CMS_CFG: cms/envs/bok_choy.yml + DJANGO_SETTINGS_MODULE: cms.envs.devstack + run: ./manage.py cms collectstatic --noinput + + - name: Generate List of Collected Staticfiles with Hashes + run: find "${{ steps.look-up-static-root.outputs.STATIC_ROOT }}" -type f | sort | xargs sha1sum > staticfiles-hashes-new.txt + + # ... | bash -c 'while read -r path; do echo "$path ::: $(sha1sum "$path")"' + + - name: Upload List of Collected Staticfiles with Hashes + uses: actions/upload-artifact@v3 + with: + name: staticfiles-hashes-new + path: staticfiles-hashes-new.txt + + compare_legacy_and_new: + name: Compare legacy and new asset outputs + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-20.04 ] + python-version: [ 3.8 ] + node-version: [ 16 ] + npm-version: [ 8.5.x ] + needs: + - static_assets_check + - static_assets_check_new + + steps: + - name: Download Hashes of Legacy Collected Staticfiles + uses: actions/download-artifact@v3 + with: + name: staticfiles-hashes-legacy + + - name: Download Hashes of New Collected Staticfiles + uses: actions/download-artifact@v3 + with: + name: staticfiles-hashes-new + + - name: Compare old and new + run: diff staticfiles-hashes-legacy.txt staticfiles-hashes-new.txt