From e597fa4b477e5da0d6a9c89c1c3b199e0f54a20c Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Tue, 17 Jan 2023 15:58:14 -0500 Subject: [PATCH 01/30] build: rewrite `paver update_assets` as shell script TODO details TODO tickets --- .../0016-build-assets-without-python.rst | 71 ++ scripts/assets.sh | 694 ++++++++++++++++++ 2 files changed, 765 insertions(+) create mode 100644 docs/decisions/0016-build-assets-without-python.rst create mode 100755 scripts/assets.sh diff --git a/docs/decisions/0016-build-assets-without-python.rst b/docs/decisions/0016-build-assets-without-python.rst new file mode 100644 index 000000000000..bf03ceb37a70 --- /dev/null +++ b/docs/decisions/0016-build-assets-without-python.rst @@ -0,0 +1,71 @@ +Building static assets without Python +##################################### + +Status +****** + +Pending + +Will be moved to *Accepted* upon completion of re-implementation. + +Context +******* + +edx-platform assets (including legacy LMS views, legacy CMS views, and built-in XBlock views) are built using Python functions in the ``pavelib/`` directory. The build process includes: + +* Copying files from ``node_modules/`` into various "vendor" directories to support RequireJS frontends. +* Collecting assets from XModule-style XBlocks that are built-in to the platform. +* Compiling common SCSS into CSS, both for base files and for theme-provided files. +* Running Webpack. +* Collecting assets into the ``STATIC_ROOT``. + +Tutor invokes these functions via its custom `openedx-assets`_ Python script. Devstack and the old Ansible installation both invoke these functions via the ``paver update_assets`` command. + +.. _openedx-assets: https://github.com/overhangio/tutor/blob/open-release/olive.1/tutor/templates/build/openedx/bin/openedx-assets. + +Updating the asset build pipeline will be necessary for several current and upcoming efforts, including: + +* `Finish upgrading frontend frameworks `_ +* `Move node_modules outside of edx-platform in Tutor's openedx image `_ +* `Move static assets outside of edx-platform in Tutor's openedx image `_ + +This has caused us to consider the value of updating the asset pipeline in place, versus rewriting and simplying it first. + +Decision +******** + +TODO + +Rationale: + + * Other parts of pavelib have already been reimplemented, like Python + unit tests. We're following that trend. + * The Python logic in pavelib is harder to understand than simple + shell scripts. + * pavelib has dependencies (Python, paver, edx-platform, other libs) + which means that any pavelib scripts must be executed later in + the edx-platform build process than we might want them to. For + example, in a Dockerfile, it might be more performant to process + npm assets *before* installing Python, but as long as we are still + using pavelib, that is not an option. + * The benefits of paver have been eclipsed by other tools, like + Docker (for requisite management) and Click (for CLI building). + * In the next couple commits, we make improvements to + process-npm-assets.sh. These improvements would have been possible + in the pavelib implementation, but would have been more complicated. +... + +Consequences +************ + +TODO + +... + +Alternatives Considered +*********************** + +TODO + +... + diff --git a/scripts/assets.sh b/scripts/assets.sh new file mode 100755 index 000000000000..08d3ee62bffb --- /dev/null +++ b/scripts/assets.sh @@ -0,0 +1,694 @@ +#!/usr/bin/env bash +ABOUT="Various assets processing/building/collection utility for Open edX" + +# Enable stricter error handling. +set -euo pipefail + +# True script name, so that we can call ourselves from watchers +THIS_SCRIPT="$0" + +# Constants, overridable via environment variables +SCRIPT_NAME="${SCRIPT_NAME:-$THIS_SCRIPT}" # Script name for logging & errors +DEFAULT_STATIC_ROOT="${DEFAULT_STATIC_ROOT:-./test_root/static}" # Fallback STATIC_ROOT (for Django settings) +DEFAULT_THEMES_DIR="${DEFAULT_THEMES_DIR:-}" # Fallback directory to look for themes in +DEFAULT_COLLECT_SETTINGS="${DEFAULT_COLLECT_SETTINGS:-lms.envs.production}" # Fallback `./manage.py collectstatic` settings + +# Codes for colored terminal output. +COL_LOG="\e[36m" # Log/step/section color (cyan) +COL_RUN="\e[35m" # Executed code echo color (purple) +COL_ERR="\e[31m" # Error color (red) +COL_OFF="\e[0m" # Normal color + +# Print script usage information. +show_usage ( ) { + echo "Usage: $SCRIPT_NAME [] []" + echo + echo "$ABOUT" + echo + echo "Subcommands:" + echo " build Build all assets (npm+xmodule+webpack+common+themes)" + echo " npm Copy static assets from node_modules" + echo " xmodule Process assets from xmodule" + echo " webpack Run webpack" + echo " common Compile static assets for common theme" + echo " themes Compile static assets for custom themes" + echo " collect Collect static assets to be served by webserver" + echo " watch-themes Watch theme assets for changes and recompile on-the-fly" + echo + echo "Options:" + echo " -h|--help Display this help message." + echo " -d|--dry-run Print shell commands but do not run them." + echo " -e|--env Environment: prod or dev. Default is prod." + echo " -r|--static-root Path for Webpack output." + echo " Default is $DEFAULT_STATIC_ROOT." + echo " -s|--settings Dotted path to Django settings module for asset" + echo " collection. Default is $DEFAULT_COLLECT_SETTINGS" + echo " --systems ... Specify one or more systems: lms, cms. Default is both." + echo " --theme-dirs ... Specify one or more theme search dirs." + echo " Default is ${DEFAULT_THEMES_DIR:-none}." + echo " --themes ... Compile one or more themes from theme search dirs." + echo " Default is all." +} + +# Print a formatted error message. +show_error ( ) { + local error_message="$1" + + echo -e "${COL_ERR}${SCRIPT_NAME}: error: ${error_message}${COL_OFF}" +} + +# Print a formatted error message, and exit the script unsuccessfully. +fail ( ) { + local error_message="$1" + + show_error "$error_message" + exit 1 +} + +# Print a formatted error message, tell the user how to view the script's usage info, +# and exit the script unsucessfully. Use this function when the error is simply +# that the script has been called wrong. +fail_usage ( ) { + local error_message="$1" + + show_error "$error_message" + echo + echo "Try '$SCRIPT_NAME -h' or '$SCRIPT_NAME --help' for more information." + exit 1 +} + +# Log the beginning of a "section" (a larger part of the script). +log_section_start ( ) { + local section_description="$*" + + echo -e "${COL_LOG}=====================================================================================$COL_OFF" + echo -e "${COL_LOG} $section_description $COL_OFF" + echo -e "${COL_LOG}-------------------------------------------------------------------------------$COL_OFF" +} + +# Log the end of a "section" (a larger part of the script). +log_section_end ( ) { + local section_description="$*" + + echo -e "${COL_LOG}-------------------------------------------------------------------------------$COL_OFF" + echo -e "${COL_LOG} $section_description $COL_OFF" + echo -e "${COL_LOG}=====================================================================================$COL_OFF" +} + +# Log a line of information. +log ( ) { + local log_line="$*" + + echo -e "${COL_LOG}$SCRIPT_NAME: $log_line $COL_OFF" +} + +# run: a 'function pointer' to either _echo_and_run_command or _echo_command. +# +# All shell commands invoked by this script that mutate the environment +# are wrapped in a call to '"$run" ...'. This gives us two benefits: +# 1. Commands are always echoed before they are executed, making it +# easier for users to understand & debug the script from its output. +# 2. If the user passses --dry-run, we set run="_echo_command", +# allowing to the script to be run with command printed but not executed. +run="_echo_and_run_command" + +# Print a shell command (without running it). +# Command should be passed as separate arguments. +_echo_command ( ) { + local command_components=("$@") + + echo -e "${COL_RUN}${command_components[*]}${COL_OFF}" +} + +# Print & execute a shell command. +# Command should be passed as separate arguments. +_echo_and_run_command ( ) { + local command_components=("$@") + + _echo_command "${command_components[@]}" + "${command_components[@]}" +} + +# Compile a directory of SCSS into CSS, generating RTL CSS as needed. +# +# TODO: Unlike its Python API, libsass-python's CLI (sassc) only supports compiling individual +# SCSS files, not entire directories. However, the CLIs for dart-sass and node-sass +# both *do* support compiling entire directories. So, if/when we upgrade to one of those +# libraries, much of this function can be replaced with a single CLI call. +compile_scss_dir ( ) { + local scss_env="$1" # 1: Environment (dev or prod). For output styling. + local scss_src_root="$2" # 2: Path to source directory containing SCSS. + local css_dest_root="$3" # 3: Path to target directory for CSS. + shift 3 + local include_paths=("$@") # Remaining args: Search paths for SCSS imports. + + local sassc_options=() + + # Add output-style option, depending on environment. + if [[ "$scss_env" == dev ]] ; then + sassc_options+=("--output-style=nested") + else + sassc_options+=("--output-style=compressed") + fi + + # For each include path, add it to the list of SCSS compile options. + for include_path in "${include_paths[@]}" ; do + sassc_options+=("--include-path=$include_path") + done + + # For each SCSS file $scss_src within $scss_src_root (recursive), + # excluding underscore-prefixed (i.e., partial) SCSS files... + while read -r -d $'\0' scss_src ; do + + # Translate source path into destination path: + scss_src_relative="${scss_src#"$scss_src_root"}" # Chop off SCSS root dir prefix. + css_dest_relative="${scss_src_relative%.scss}.css" # Replace file extension. + css_dest="$css_dest_root/$css_dest_relative" # Prepend CSS root dir. + + css_dest_dir="$(dirname "$css_dest")" # Find immediate parent dir of CSS file target... + "$run" mkdir -p "$css_dest_dir" # ...and create it if it doesn't exist. + "$run" sassc "${sassc_options[@]}" "$scss_src" "$css_dest" # Compile the SCSS. + + # If this is an LTR (left-to-right) SCSS source file... + if [[ "$scss_src" != *-rtl.scss ]] ; then + + # then determine what the name of the RTL source and target would be... + rtl="$scss_src_relative" # (Start with the SCSS-root-relative source path, + rtl="${rtl%-ltr.scss}" # then strip any -ltr.scss suffix, + rtl="${rtl%.scss}" # as well as any strip any .scss suffix. + rtl="${rtl}-rtl" # then finally append -rtl) + rtl_scss_src="$scss_src_root/$rtl.scss" + rtl_css_dest="$css_dest_root/$rtl.css" + + # and if the source RTL SCSS doesn't exist... + if [[ ! -f "$rtl_scss_src" ]] ; then + + # then we know that the target RTL CSS will not be generated via SCSS compilation, + # so we must auto-generate it here from the LTR CSS. + "$run" rtlcss "$css_dest" "$rtl_css_dest" + fi + fi + done < <(find "$scss_src_root" -type f -name '*.scss' \! -name '_*' -print0) +} + +# Variables, controlled by command-line options. + +subcommand="" +env="prod" + +static_root_lms="$DEFAULT_STATIC_ROOT" +static_root_cms="$static_root_lms/studio" +collect_django_settings="$DEFAULT_COLLECT_SETTINGS" + +theme_names=() +if [[ -n "$DEFAULT_THEMES_DIR" ]] ; then + theme_dirs=("$DEFAULT_THEMES_DIR") +else + theme_dirs=() +fi + +# In https://github.com/openedx/wg-developer-experience/issues/150, +# we will allow this to be configured via a new --node-modules option. +node_modules="node_modules" + +# "Boolean" variables, controlled by command-line options. +# Non-empty string is "True", empty string is "False". +# As a convention, we use the string "T" for "True". + +do_lms="T" +do_cms="T" + +# Arguments can take a few different forms, for better or for worse: +# * Positional arguments (not prefixed with '-') +# * Flag options, both short (-d) or long (--do-stuff) +# * Grouped short flag options (-dxy) +# * Single-value options, both short (-k val, -k=val) and long (--key val, --key=val) +# * Multi-value options, both short (-k val1 val2 val3) and long (--key val1 val2 val3) +# To simplify the next part of the script, we simplify the args: +# * Expand grouped short flags into individual short values (-abc => -a -b -c) +# * Remove the equals sign from value options (--key=val => --key val) +simplified_args=() +while [[ "$#" -gt 0 ]] ; do + case "$1" in + --*) + long_option="${1#--}" + if [[ "$long_option" == *'='* ]] ; then + # Long option using equals sign. + # Split it into key and value as two separate args. + long_option_key="$(echo "$long_option" | cut -d '=' -f 1)" + if [[ -z "${long_option_key}" ]] ; then + # If the key is empty, something's wrong ('-=') + fail_usage "Bad option: $1" + fi + long_option_val="$(echo "$long_option" | cut -d '=' -f 2-)" + simplified_args+=("--$long_option_key" "$long_option_val") + else + # Long option with no equals sign. Good as-is. + simplified_args+=("$1") + fi + ;; + -*) + short_option="${1#-}" + if [[ "$short_option" == *'='* ]] ; then + # Short option using equals sign. + # Split it into key and value as two separate args. + short_option_key="$(echo "$short_option" | cut -d '=' -f 1)" + if [[ "${#short_option_key}" != 1 ]] ; then + # If the key isn't one character, something's wrong ('-key=val') + fail_usage "Bad option: $1" + fi + short_option_val="$(echo "$short_option" | cut -d '=' -f 2-)" + simplified_args+=("-$short_option_key" "$short_option_val") + else + # Short option(s) with no equals sign. + # Treat each character as its own short option. + for (( i=0; i<${#short_option}; i++ )); do + simplified_args+=("-${short_option:i:1}") + done + fi + ;; + + *) + # Positional argument, or value for an option. Good as-is. + simplified_args+=("$1") + ;; + esac + shift +done + +# Now that we've simplified the arguments into a consistent format, +# loop through and process them. +set -- "${simplified_args[@]}" +while [[ "$#" -gt 0 ]] ; do + case "$1" in + + -h|--help) + show_usage + exit 0 + ;; + + -d|--dry) + run="_echo_command" + log "DRY RUN: Commands will be printed but not executed!" + shift + ;; + + -e|--env) + case "$2" in + dev|prod) + env="$2" + ;; + *) + fail_usage "expected prod or dev, got: $2" + ;; + esac + env="$2" + shift 2 + ;; + + -r|--static-root) + if [[ "$#" -eq 1 ]] ; then + fail_usage "Missing value for $1" + fi + static_root_lms="$2" + static_root_cms="$2/studio" + shift 2 + ;; + + --themes) + shift + # Append args as theme dirs until we hit an option ( -* ). + while [[ "$#" -gt 0 ]] && ! [[ "$1" = -* ]]; do + theme_names+=("$1") + shift + done + ;; + + --theme-dirs) + shift + # Append args as theme dirs until we hit an option ( -* ). + theme_dirs=() + while [[ "$#" -gt 0 ]] && [[ "$1" != -* ]]; do + theme_dirs+=("$1") + shift + done + ;; + + --systems) + shift + do_lms="" + do_cms="" + # Treat args as systems until we hit an option ( -* ). + while [[ "$#" -gt 0 ]] && ! [[ "$1" = -* ]]; do + case "$1" in + lms) + do_lms="T" + ;; + cms) + do_cms="T" + ;; + *) + fail_usage "Valid systems are: lms, cms. Got: $1" + ;; + esac + shift + done + if [[ -z "$do_lms" ]] && [[ -z "$do_cms" ]] ; then + fail_usage "You must specify one or more system" + fi + ;; + + -s|--settings) + collect_django_settings="$2" + shift 2 + ;; + + build|npm|xmodule|webpack|common|themes|collect|watch-themes) + if [[ -z "$subcommand" ]] ; then + subcommand="$1" + else + fail_usage "Cannot specify a second subcommand: $1" + fi + shift + ;; + + -*) + fail_usage "Unrecognized option: $1" + ;; + *) + fail_usage "Unexpected argument: $1" + ;; + esac +done + +if [[ -z "$subcommand" ]] ; then + fail_usage "Please specify a subcommand" +fi + +if [[ "$subcommand" = build ]] || [[ "$subcommand" = npm ]] ; then + + log_section_start "Copying static assets from node_modules..." + + # Vendor destination paths for assets. + # These are not configurable yet, but that will change as part of + # https://github.com/openedx/wg-developer-experience/issues/151 + js_vendor_path="common/static/common/js/vendor" + css_vendor_path="common/static/common/css/vendor" + edx_ui_toolkit_vendor_path="common/static/edx-ui-toolkit" + + log "Ensuring vendor directories exist..." + "$run" mkdir -p "$js_vendor_path" + "$run" mkdir -p "$css_vendor_path" + "$run" mkdir -p "$edx_ui_toolkit_vendor_path" + + log "Copying studio-frontend JS & CSS from node_modules into vendor directores..." + find "$node_modules/@edx/studio-frontend/dist" -type f -print0 | \ + while read -r -d $'\0' src_file ; do + if [[ "$src_file" = *.css ]] || [[ "$src_file" = *.css.map ]] ; then + "$run" cp --force "$src_file" "$css_vendor_path" + else + "$run" cp --force "$src_file" "$js_vendor_path" + fi + done + + log "Copying certain JS modules from node_modules into vendor directory..." + js_vendor_modules=( + "$node_modules/backbone.paginator/lib/backbone.paginator.js" + "$node_modules/backbone/backbone.js" + "$node_modules/bootstrap/dist/js/bootstrap.bundle.js" + "$node_modules/hls.js/dist/hls.js" + "$node_modules/jquery-migrate/dist/jquery-migrate.js" + "$node_modules/jquery.scrollto/jquery.scrollTo.js" + "$node_modules/jquery/dist/jquery.js" + "$node_modules/moment-timezone/builds/moment-timezone-with-data.js" + "$node_modules/moment/min/moment-with-locales.js" + "$node_modules/picturefill/dist/picturefill.js" + "$node_modules/requirejs/require.js" + "$node_modules/underscore.string/dist/underscore.string.js" + "$node_modules/underscore/underscore.js" + "$node_modules/which-country/index.js" + ) + for js_vendor_module in "${js_vendor_modules[@]}" ; do + "$run" cp --force "$js_vendor_module" "$js_vendor_path" + done + + log "Copying certain JS developer modules into vendor directory..." + if [[ "$env" = dev ]] ; then + "$run" cp --force "$node_modules/sinon/pkg/sinon.js" "$js_vendor_path" + "$run" cp --force "$node_modules/squirejs/src/Squire.js" "$js_vendor_path" + else + # TODO: https://github.com/openedx/edx-platform/issues/31768 + # In the old implementation of this scipt (pavelib/assets.py), these two + # developer libraries were copied into the JS vendor directory whether not + # the build was for prod or dev. In order to exactly match the output of + # the old script, this script will also copy them in for prod builds. + # However, in the future, it would be good to only copy them for dev + # builds. Furthermore, these libraries should not be `npm install`ed + # into prod builds in the first place. + "$run" cp --force "$node_modules/sinon/pkg/sinon.js" "$js_vendor_path" || true # "|| true" means "tolerate errors"; in this case, + "$run" cp --force "$node_modules/squirejs/src/Squire.js" "$js_vendor_path" || true # that's "tolerate if these files don't exist." + fi + + log_section_end "Done copying static assets from node_modules." +fi + +if [[ "$subcommand" = build ]] || [[ "$subcommand" = xmodule ]] ; then + + log_section_start "Processing assets from xmodule..." + + # Note: + # Collecting xmodule_assets is incompatible with setting the django path because + # of an unfortunate call to settings.configure(), so we must clear + # DJANGO_SETTINGS_MODULE before calling the script. + + "$run" env \ + "DJANGO_SETTINGS_MODULE=" \ + xmodule_assets common/static/xmodule + + log_section_end "Done processing assets from xmodule." +fi + +if [[ "$subcommand" = build ]] || [[ "$subcommand" = webpack ]] ; then + + log_section_start "Running webpack..." + + node_env="production" + if [[ "$env" = dev ]]; then + node_env="development" + fi + + "$run" env \ + "NODE_ENV=$node_env" \ + "STATIC_ROOT_LMS=$static_root_lms" \ + "STATIC_ROOT_CMS=$static_root_cms" \ + webpack --progress "--config=webpack.$env.config.js" + + log_section_end "Done running webpack." +fi + +# SCSS source roots. +lms_scss="lms/static/sass" +cms_scss="cms/static/sass" +certs_scss="lms/static/certificates/sass" + +# SCSS dependency roots (include lists are order-sensitive!) +lms_partials="lms/static/sass/partials" +cms_partials="cms/static/sass/partials" +common_includes=( + "common/static" + "common/static/sass" + "$node_modules" + "$node_modules/@edx" +) +lms_includes=( + "${common_includes[@]}" + "$lms_partials" + "$lms_scss" +) +cms_includes=( + "${common_includes[@]}" + "$lms_partials" + "$cms_partials" + "$cms_scss" +) +certs_includes=("${lms_includes[@]}") + +# CSS target roots. +lms_css="lms/static/css" +cms_css="cms/static/css" +certs_css="lms/static/certificates/css" + + +if [[ "$subcommand" = build ]] || [[ "$subcommand" = common ]] ; then + + log_section_start "Compiling static assets for common theme..." + + if [[ -n "$do_lms" ]] ; then + log "Compiling default LMS SCSS." + compile_scss_dir "$env" "$lms_scss" "$lms_css" "${lms_includes[@]}" + log "Compiling default certificates SCSS." + compile_scss_dir "$env" "$certs_scss" "$certs_css" "${certs_includes[@]}" + fi + if [[ -n "$do_cms" ]] ; then + log "Compiling default CMS SCSS." + compile_scss_dir "$env" "$cms_scss" "$cms_css" "${cms_includes[@]}" + fi + + log_section_end "Done compiling static assets for common theme." +fi + +theme_paths=() +for theme_dir in "${theme_dirs[@]}" ; do + for theme_dir_item in "$theme_dir"/* ; do + if [[ ! -d "$theme_dir_item" ]] ; then + continue + fi + if [[ "${#theme_names[@]}" = 0 ]] ; then + theme_paths+=("$theme_dir_item") + continue + fi + for theme_name in "${theme_names[@]}" ; do + if [[ "$(basename "$theme_dir_item")" = "$theme_name" ]] ; then + theme_paths+=("$theme_dir_item") + break + fi + done + done +done + +if [[ "$subcommand" = build ]] || [[ "$subcommand" = themes ]] ; then + + for theme_path in "${theme_paths[@]}" ; do + + log_section_start "Compiling static assets for custom theme: $theme_path..." + + # Theme SCSS source roots. + theme_lms_scss="$theme_path/lms/static/sass" + theme_cms_scss="$theme_path/cms/static/sass" + theme_certs_scss="$theme_path/lms/static/certificates/sass" + + # Theme SCSS dependency roots (include lists are order-sensitive!) + theme_cms_partials="$theme_path/cms/static/sass/partials" + theme_lms_partials="$theme_path/lms/static/sass/partials" + theme_lms_includes=( + "${common_includes[@]}" + "$theme_lms_partials" + "$lms_partials" + "$lms_scss" + ) + theme_cms_includes=( + "${common_includes[@]}" + "$lms_partials" + "$theme_cms_partials" + "$cms_partials" + "$cms_scss" + ) + theme_certs_includes=( + "${common_includes[@]}" + "$theme_lms_partials" + "$theme_lms_scss" + ) + + # Theme CSS target roots. + theme_lms_css="$theme_path/lms/static/css" + theme_cms_css="$theme_path/cms/static/css" + theme_certs_css="$theme_path/lms/static/certificates/css" + + if [[ -n "$do_lms" ]] ; then + if [[ -d "$theme_lms_scss" ]] ; then + log "Compiling default LMS SCSS into theme's CSS directory." + compile_scss_dir "$env" "$lms_scss" "$theme_lms_css" "${theme_lms_includes[@]}" + log "Compiling theme's LMS SCSS into theme's CSS directory." + compile_scss_dir "$env" "$theme_lms_scss" "$theme_lms_css" "${theme_lms_includes[@]}" + else + log "Theme has no LMS SCSS; skipping." + fi + if [[ -d "$theme_certs_scss" ]] ; then + log "Compiling theme's certificate SCSS into theme's CSS directory." + compile_scss_dir "$env" "$theme_certs_scss" "$theme_certs_css" "${theme_certs_includes[@]}" + else + log "Theme has no certificate SCSS; skipping." + fi + fi + if [[ -n "$do_cms" ]] ; then + if [[ -d "$theme_cms_scss" ]] ; then + log "Compiling default CMS SCSS into theme's CSS directory." + compile_scss_dir "$env" "$cms_scss" "$theme_cms_css" "${theme_cms_includes[@]}" + log "Compiling theme's CMS SCSS into theme's CSS directory." + compile_scss_dir "$env" "$theme_cms_scss" "$theme_cms_css" "${theme_cms_includes[@]}" + else + log "Theme has no CMS SCSS; skipping." + fi + fi + + log_section_end "Done compiling theme: $theme_path" + done + +fi + +if [[ "$subcommand" = collect ]] ; then + + log_section_start "Collecting static assets to be served by webserver..." + + if [[ -n "$do_lms" ]] ; then + "$run" ./manage.py lms collectstatic --noinput \ + --settings "$collect_django_settings" \ + --ignore 'fixtures' \ + --ignore 'karma_*.js' \ + --ignore 'spec' \ + --ignore 'spec_helpers' \ + --ignore 'spec-helpers' \ + --ignore 'xmodule_js' \ + --ignore 'geoip' \ + --ignore 'sass' + fi + if [[ -n "$do_cms" ]] ; then + "$run" ./manage.py cms collectstatic --noinput \ + --settings "$collect_django_settings" \ + --ignore 'fixtures' \ + --ignore 'karma_*.js' \ + --ignore 'spec' \ + --ignore 'spec_helpers' \ + --ignore 'spec-helpers' \ + --ignore 'xmodule_js' \ + --ignore 'geoip' \ + --ignore 'sass' + fi + + log_section_end "Done collecting static assets to be served by webserver." +fi + +if [[ "$subcommand" = watch-themes ]] ; then + + # TODO: implement watchers, both for themes and for other assets + fail "Watchers do not work yet." + + log_section_start "Starting watchers for theme assets..." + cleanup="echo 'Cleaning up watchers...'" + + for theme_path in "${theme_paths[@]}" ; do + theme_dir="$(dirname "$theme_path")" + theme_name="$(basename "$theme_path")" + for system in lms cms ; do + watch_path="$theme_path/$system" + watchman "--logfile=$(realpath ..)/watchman.log" watch "$watch_path" + watchman -- trigger "$watch_path" "recompile-scss-on-change--$watch_path" '*scss' -- \ + "$THIS_SCRIPT" themes \ + --theme-dirs "$theme_dir" \ + --themes "$theme_name" \ + --systems "$system" + cleanup="$cleanup ; watchman watch-del '$watch_path'" + done + done + + cleanup="$cleanup ; watchman shutdown-server ; echo 'Done cleaning up watchers.'" + # Shellcheck will warn us that "$cleanup" will be evaulated now, not + # when the signal is trapped. In fact, that's exactly what we want. + # shellcheck disable=SC2064 + trap "$cleanup" EXIT INT HUP TERM + + log_section_end "Watchers started. Use Ctrl-c to exit." + while true ; do + sleep 1 + done +fi From 1c03e46ce00f01e6e5b6c79dba1cf816f68257d9 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 16 Feb 2023 15:46:51 -0500 Subject: [PATCH 02/30] squash: adr wip --- docs/decisions/0016-assets-without-python.rst | 150 ++++++++++++++++++ .../0016-build-assets-without-python.rst | 71 --------- 2 files changed, 150 insertions(+), 71 deletions(-) create mode 100644 docs/decisions/0016-assets-without-python.rst delete mode 100644 docs/decisions/0016-build-assets-without-python.rst diff --git a/docs/decisions/0016-assets-without-python.rst b/docs/decisions/0016-assets-without-python.rst new file mode 100644 index 000000000000..8e52e4668810 --- /dev/null +++ b/docs/decisions/0016-assets-without-python.rst @@ -0,0 +1,150 @@ +Building static assets without Python +##################################### + +Status +****** + +Pending + +Will be moved to *Accepted* upon completion of re-implementation. + +Context +******* + +State of edx-platform frontends +=============================== + +New Open edX frontend development has largely moved to React-based micro-frontends (MFEs). However, edx-platform still has a few categories of important static frontend assets: + +.. list-table:: + :header-rows: 1 + +* - **Name** + - Description + - Example + - Expected direction +* - **XBlock Fragments** + - JS and CSS belonging to the pure XBlocks defined in edx-platform + - library_sourced_block.js + - Keep, or extract to per-XBlock repositories +* - **XModule Fragments** + - JS and SCSS belonging to the older XModule-style XBlocks defined in edx-platform + - ProblemBlock (aka CAPA) assets + - Convert to pure XBlock fragments +* - **Legacy LMS Frontends** + - JS, SCSS, and other resources powering LMS views that have not yet been replatformed into MFEs + - Instructor Dashboard assets + - Replatform & DEPR +* - **Legacy CMS Frontends** + - JS, SCSS, and other resources powering Studio views that have not yet been replatformed into MFEs + - Course outline editor and unit editor + - Replatform & DEPR +* - **Shared Frontend Files** + - JS modules, SCSS partials, and other resources, usable by both Legacy LMS and CMS Frontends. This includes a few libraries that have been committed to edx-platform in their entirety. + - Legacy cookie policy banner; CodeMirror + - Remove as part of full LMS/CMS frontend replatforming +* - **pip-installed Assets** + - Pre-compiled static assets shipped with several Python libraries that we install, including XBlocks. Not committed to edx-platform. + - Django Admin, Swagger, Drag-And-Drop XBlock V2 + - Keep +* - **npm-installed Assets** + - JS modules and CSS files installed via NPM. Not committed to edx-platform. + - React + - Remove as part of full LMS/CMS frontend replatforming + +(Note that this table excludes HTML templates. Templates are part of the frontend, but they are dynamically rendered by the Web application and therefore must be handled differently than static assets.) + +So with the exception of XBlock fragments and pip-installed assets, which are very simple for edx-platform to handle, we plan to eventually remove all edx-platform static frontend assets. However, given the number of remaining edx-platform frontends and speed at which they are currently being replatformed, estimates for completion of this process range from one to five years. Thus, in the medium term future, we feel that timeboxed improvements to how edx-platform handles static assets are worthwhile. + +Types of asset processing +========================= + +There are three actions a developer or a deployment pipeline may need to take on edx-platform static assets: + +* **Build:** Compile, generate, copy, and otherwise process static assets so that they can be used by the Django webserver or collected elsewhere. For many Web applications, all static asset building would be coordinated via Webpack or another NPM-managed tool. Due to the age of edx-platform and its legacy XModule and Comprehensive Themeing systems, though, there are five specific build steps, which generally need to be performed in this order: + + 1. **Copy NPM-installed assets** from node_modules to places where they can be used by certain especially-old edx-platform frontends that do not work with NPM. + * **Copy XModule assets** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. + 2. **Run Webpack** to shim, minify, and bundle JS modules. + 4. **Compile Default SCSS** into CSS. + 5. **Compile Theme SCSS** into CSS from some number of operator-specified theme directories, using default SCSS as a base. + +* **Collect:** Copy static assets from edx-platform to another location (the ``STATIC_ROOT``) so that they can be efficiently served *without* Django's webserver. This step, by nature, requires Python and Django in order to find and organize the assets, which may come from edx-platform itself or from its many installed Python and NPM packages. This is only done for production environments, where it is usually desirable to serve assets with something efficient like NGINX. + +* **Watch:** Listen for changes to static assets in the background. When a change occurs, rebuild them automatically, so that the Django webserver picks up the changes. This is only necessary in development environments. A few different sets of assets can be watched: + * XModule assets. Upon change, these should be re-copied, which should trigger a Webpack re-run and a defualt SCSS recompilation. + * JavaScript modules. Upon change, a Webpack re-run should be triggered. + * Default SCSS. Upon change, it should be re-compiled, as should theme SCSS. + * Theme SCSS. Upon change, it should be re-compiled. + +Entry points for asset processing +================================= + +Today, there are two main ways an operators would perform these actions: + +* via edx-platform's ``paver`` command-line interface (defined in the `pavelib`_ source tree), which wraps all the actions in Python, and requires Django. Example usage, via Devstack:: + + make lms-shell + paver update_assets + +* via the `openedx-assets`_ script, which Tutor adds to LMS and CMS containers. It uses a mix of its own Python wrapper code and calls to the pavelib implementation mentioned above. It avoids parts of pavelib that Tutor's authors found slow or buggy. Example usage:: + + tutor dev run lms openedx-assets --env=dev + +Python used in the asset build +============================== + + + +Etc +=== + +.. _paver: https://github.com/openedx/tutor/tree/open-release/olive.1/pavelib +.. _openedx-assets: https://github.com/overhangio/tutor/blob/v15.0.0/tutor/templates/build/openedx/bin/openedx-assets. + +Updating the asset build pipeline will be necessary for several current and upcoming efforts, including: + +* `Finish upgrading frontend frameworks `_ +* `Move node_modules outside of edx-platform in Tutor's openedx image `_ +* `Move static assets outside of edx-platform in Tutor's openedx image `_ + +This has caused us to consider the value of updating the asset pipeline in place, versus rewriting and simplying it first. + +Decision +******** + +TODO + +Rationale: + + * Other parts of pavelib have already been reimplemented, like Python + unit tests. We're following that trend. + * The Python logic in pavelib is harder to understand than simple + shell scripts. + * pavelib has dependencies (Python, paver, edx-platform, other libs) + which means that any pavelib scripts must be executed later in + the edx-platform build process than we might want them to. For + example, in a Dockerfile, it might be more performant to process + npm assets *before* installing Python, but as long as we are still + using pavelib, that is not an option. + * The benefits of paver have been eclipsed by other tools, like + Docker (for requisite management) and Click (for CLI building). + * In the next couple commits, we make improvements to + process-npm-assets.sh. These improvements would have been possible + in the pavelib implementation, but would have been more complicated. +... + +Consequences +************ + +TODO + +... + +Alternatives Considered +*********************** + +TODO + +... + diff --git a/docs/decisions/0016-build-assets-without-python.rst b/docs/decisions/0016-build-assets-without-python.rst deleted file mode 100644 index bf03ceb37a70..000000000000 --- a/docs/decisions/0016-build-assets-without-python.rst +++ /dev/null @@ -1,71 +0,0 @@ -Building static assets without Python -##################################### - -Status -****** - -Pending - -Will be moved to *Accepted* upon completion of re-implementation. - -Context -******* - -edx-platform assets (including legacy LMS views, legacy CMS views, and built-in XBlock views) are built using Python functions in the ``pavelib/`` directory. The build process includes: - -* Copying files from ``node_modules/`` into various "vendor" directories to support RequireJS frontends. -* Collecting assets from XModule-style XBlocks that are built-in to the platform. -* Compiling common SCSS into CSS, both for base files and for theme-provided files. -* Running Webpack. -* Collecting assets into the ``STATIC_ROOT``. - -Tutor invokes these functions via its custom `openedx-assets`_ Python script. Devstack and the old Ansible installation both invoke these functions via the ``paver update_assets`` command. - -.. _openedx-assets: https://github.com/overhangio/tutor/blob/open-release/olive.1/tutor/templates/build/openedx/bin/openedx-assets. - -Updating the asset build pipeline will be necessary for several current and upcoming efforts, including: - -* `Finish upgrading frontend frameworks `_ -* `Move node_modules outside of edx-platform in Tutor's openedx image `_ -* `Move static assets outside of edx-platform in Tutor's openedx image `_ - -This has caused us to consider the value of updating the asset pipeline in place, versus rewriting and simplying it first. - -Decision -******** - -TODO - -Rationale: - - * Other parts of pavelib have already been reimplemented, like Python - unit tests. We're following that trend. - * The Python logic in pavelib is harder to understand than simple - shell scripts. - * pavelib has dependencies (Python, paver, edx-platform, other libs) - which means that any pavelib scripts must be executed later in - the edx-platform build process than we might want them to. For - example, in a Dockerfile, it might be more performant to process - npm assets *before* installing Python, but as long as we are still - using pavelib, that is not an option. - * The benefits of paver have been eclipsed by other tools, like - Docker (for requisite management) and Click (for CLI building). - * In the next couple commits, we make improvements to - process-npm-assets.sh. These improvements would have been possible - in the pavelib implementation, but would have been more complicated. -... - -Consequences -************ - -TODO - -... - -Alternatives Considered -*********************** - -TODO - -... - From 86fa51f3b7e021ba45c1184779d8e540bb0229d2 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 16 Feb 2023 15:49:12 -0500 Subject: [PATCH 03/30] squash: rst table --- docs/decisions/0016-assets-without-python.rst | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/docs/decisions/0016-assets-without-python.rst b/docs/decisions/0016-assets-without-python.rst index 8e52e4668810..d36e03cde790 100644 --- a/docs/decisions/0016-assets-without-python.rst +++ b/docs/decisions/0016-assets-without-python.rst @@ -19,38 +19,38 @@ New Open edX frontend development has largely moved to React-based micro-fronten .. list-table:: :header-rows: 1 -* - **Name** - - Description - - Example - - Expected direction -* - **XBlock Fragments** - - JS and CSS belonging to the pure XBlocks defined in edx-platform - - library_sourced_block.js - - Keep, or extract to per-XBlock repositories -* - **XModule Fragments** - - JS and SCSS belonging to the older XModule-style XBlocks defined in edx-platform - - ProblemBlock (aka CAPA) assets - - Convert to pure XBlock fragments -* - **Legacy LMS Frontends** - - JS, SCSS, and other resources powering LMS views that have not yet been replatformed into MFEs - - Instructor Dashboard assets - - Replatform & DEPR -* - **Legacy CMS Frontends** - - JS, SCSS, and other resources powering Studio views that have not yet been replatformed into MFEs - - Course outline editor and unit editor - - Replatform & DEPR -* - **Shared Frontend Files** - - JS modules, SCSS partials, and other resources, usable by both Legacy LMS and CMS Frontends. This includes a few libraries that have been committed to edx-platform in their entirety. - - Legacy cookie policy banner; CodeMirror - - Remove as part of full LMS/CMS frontend replatforming -* - **pip-installed Assets** - - Pre-compiled static assets shipped with several Python libraries that we install, including XBlocks. Not committed to edx-platform. - - Django Admin, Swagger, Drag-And-Drop XBlock V2 - - Keep -* - **npm-installed Assets** - - JS modules and CSS files installed via NPM. Not committed to edx-platform. - - React - - Remove as part of full LMS/CMS frontend replatforming + * - **Name** + - Description + - Example + - Expected direction + * - **XBlock Fragments** + - JS and CSS belonging to the pure XBlocks defined in edx-platform + - library_sourced_block.js + - Keep, or extract to per-XBlock repositories + * - **XModule Fragments** + - JS and SCSS belonging to the older XModule-style XBlocks defined in edx-platform + - ProblemBlock (aka CAPA) assets + - Convert to pure XBlock fragments + * - **Legacy LMS Frontends** + - JS, SCSS, and other resources powering LMS views that have not yet been replatformed into MFEs + - Instructor Dashboard assets + - Replatform & DEPR + * - **Legacy CMS Frontends** + - JS, SCSS, and other resources powering Studio views that have not yet been replatformed into MFEs + - Course outline editor and unit editor + - Replatform & DEPR + * - **Shared Frontend Files** + - JS modules, SCSS partials, and other resources, usable by both Legacy LMS and CMS Frontends. This includes a few libraries that have been committed to edx-platform in their entirety. + - Legacy cookie policy banner; CodeMirror + - Remove as part of full LMS/CMS frontend replatforming + * - **pip-installed Assets** + - Pre-compiled static assets shipped with several Python libraries that we install, including XBlocks. Not committed to edx-platform. + - Django Admin, Swagger, Drag-And-Drop XBlock V2 + - Keep + * - **npm-installed Assets** + - JS modules and CSS files installed via NPM. Not committed to edx-platform. + - React + - Remove as part of full LMS/CMS frontend replatforming (Note that this table excludes HTML templates. Templates are part of the frontend, but they are dynamically rendered by the Web application and therefore must be handled differently than static assets.) @@ -94,7 +94,7 @@ Today, there are two main ways an operators would perform these actions: Python used in the asset build ============================== - +. Etc === From 747fb1bd93cb55002911f33d3bf813b7b9cb0da7 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 16 Feb 2023 15:49:49 -0500 Subject: [PATCH 04/30] squash: renumber adr --- ...thout-python.rst => 0017-assets-without-python.rst} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename docs/decisions/{0016-assets-without-python.rst => 0017-assets-without-python.rst} (96%) diff --git a/docs/decisions/0016-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst similarity index 96% rename from docs/decisions/0016-assets-without-python.rst rename to docs/decisions/0017-assets-without-python.rst index d36e03cde790..3c04c71b9477 100644 --- a/docs/decisions/0016-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -63,11 +63,11 @@ There are three actions a developer or a deployment pipeline may need to take on * **Build:** Compile, generate, copy, and otherwise process static assets so that they can be used by the Django webserver or collected elsewhere. For many Web applications, all static asset building would be coordinated via Webpack or another NPM-managed tool. Due to the age of edx-platform and its legacy XModule and Comprehensive Themeing systems, though, there are five specific build steps, which generally need to be performed in this order: - 1. **Copy NPM-installed assets** from node_modules to places where they can be used by certain especially-old edx-platform frontends that do not work with NPM. - * **Copy XModule assets** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. - 2. **Run Webpack** to shim, minify, and bundle JS modules. - 4. **Compile Default SCSS** into CSS. - 5. **Compile Theme SCSS** into CSS from some number of operator-specified theme directories, using default SCSS as a base. + #. **Copy NPM-installed assets** from node_modules to places where they can be used by certain especially-old edx-platform frontends that do not work with NPM. + # **Copy XModule assets** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. + #. **Run Webpack** to shim, minify, and bundle JS modules. + #. **Compile Default SCSS** into CSS. + #. **Compile Theme SCSS** into CSS from some number of operator-specified theme directories, using default SCSS as a base. * **Collect:** Copy static assets from edx-platform to another location (the ``STATIC_ROOT``) so that they can be efficiently served *without* Django's webserver. This step, by nature, requires Python and Django in order to find and organize the assets, which may come from edx-platform itself or from its many installed Python and NPM packages. This is only done for production environments, where it is usually desirable to serve assets with something efficient like NGINX. From 9dc5ca67050fe56a0c3152932b31ba9b9d3a6d8e Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 16 Feb 2023 15:53:41 -0500 Subject: [PATCH 05/30] squash: spacing --- docs/decisions/0017-assets-without-python.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index 3c04c71b9477..6751d77ad837 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -64,17 +64,25 @@ There are three actions a developer or a deployment pipeline may need to take on * **Build:** Compile, generate, copy, and otherwise process static assets so that they can be used by the Django webserver or collected elsewhere. For many Web applications, all static asset building would be coordinated via Webpack or another NPM-managed tool. Due to the age of edx-platform and its legacy XModule and Comprehensive Themeing systems, though, there are five specific build steps, which generally need to be performed in this order: #. **Copy NPM-installed assets** from node_modules to places where they can be used by certain especially-old edx-platform frontends that do not work with NPM. + # **Copy XModule assets** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. + #. **Run Webpack** to shim, minify, and bundle JS modules. + #. **Compile Default SCSS** into CSS. + #. **Compile Theme SCSS** into CSS from some number of operator-specified theme directories, using default SCSS as a base. * **Collect:** Copy static assets from edx-platform to another location (the ``STATIC_ROOT``) so that they can be efficiently served *without* Django's webserver. This step, by nature, requires Python and Django in order to find and organize the assets, which may come from edx-platform itself or from its many installed Python and NPM packages. This is only done for production environments, where it is usually desirable to serve assets with something efficient like NGINX. * **Watch:** Listen for changes to static assets in the background. When a change occurs, rebuild them automatically, so that the Django webserver picks up the changes. This is only necessary in development environments. A few different sets of assets can be watched: + * XModule assets. Upon change, these should be re-copied, which should trigger a Webpack re-run and a defualt SCSS recompilation. + * JavaScript modules. Upon change, a Webpack re-run should be triggered. + * Default SCSS. Upon change, it should be re-compiled, as should theme SCSS. + * Theme SCSS. Upon change, it should be re-compiled. Entry points for asset processing From 643adaf979f26b3ac37952263c760a3d9a9f7110 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 16 Feb 2023 15:54:19 -0500 Subject: [PATCH 06/30] squash: --- docs/decisions/0017-assets-without-python.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index 6751d77ad837..dafef224b47b 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -65,7 +65,7 @@ There are three actions a developer or a deployment pipeline may need to take on #. **Copy NPM-installed assets** from node_modules to places where they can be used by certain especially-old edx-platform frontends that do not work with NPM. - # **Copy XModule assets** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. + #. **Copy XModule assets** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. #. **Run Webpack** to shim, minify, and bundle JS modules. From bbaddeead9db0e291ce014e08bde7d2943285bd1 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 16 Feb 2023 15:59:28 -0500 Subject: [PATCH 07/30] squash: adr work --- docs/decisions/0017-assets-without-python.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index dafef224b47b..2916a640bcbb 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -63,15 +63,15 @@ There are three actions a developer or a deployment pipeline may need to take on * **Build:** Compile, generate, copy, and otherwise process static assets so that they can be used by the Django webserver or collected elsewhere. For many Web applications, all static asset building would be coordinated via Webpack or another NPM-managed tool. Due to the age of edx-platform and its legacy XModule and Comprehensive Themeing systems, though, there are five specific build steps, which generally need to be performed in this order: - #. **Copy NPM-installed assets** from node_modules to places where they can be used by certain especially-old edx-platform frontends that do not work with NPM. + #. **Copy npm-installed assets** from node_modules to places where they can be used by certain especially-old edx-platform frontends that do not work with NPM. - #. **Copy XModule assets** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. + #. **Copy XModule Fragments** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. #. **Run Webpack** to shim, minify, and bundle JS modules. - #. **Compile Default SCSS** into CSS. + #. **Compile Default SCSS** for legacy LMS and CMS frontends into CSS. - #. **Compile Theme SCSS** into CSS from some number of operator-specified theme directories, using default SCSS as a base. + #. **Compile Theme SCSS** for legacy LMS and CMS frontends into CSS. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. * **Collect:** Copy static assets from edx-platform to another location (the ``STATIC_ROOT``) so that they can be efficiently served *without* Django's webserver. This step, by nature, requires Python and Django in order to find and organize the assets, which may come from edx-platform itself or from its many installed Python and NPM packages. This is only done for production environments, where it is usually desirable to serve assets with something efficient like NGINX. From 809cde8402fa0b0dd2ac000983440c0ae2c64dc4 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 16 Feb 2023 16:59:44 -0500 Subject: [PATCH 08/30] squash: adr problem/solutions table --- docs/decisions/0017-assets-without-python.rst | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index 2916a640bcbb..ccbc4f85009d 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -54,20 +54,33 @@ New Open edX frontend development has largely moved to React-based micro-fronten (Note that this table excludes HTML templates. Templates are part of the frontend, but they are dynamically rendered by the Web application and therefore must be handled differently than static assets.) -So with the exception of XBlock fragments and pip-installed assets, which are very simple for edx-platform to handle, we plan to eventually remove all edx-platform static frontend assets. However, given the number of remaining edx-platform frontends and speed at which they are currently being replatformed, estimates for completion of this process range from one to five years. Thus, in the medium term future, we feel that timeboxed improvements to how edx-platform handles static assets are worthwhile. +So, with the exception of XBlock fragments and pip-installed assets, which are very simple for edx-platform to handle, we plan to eventually remove all edx-platform static frontend assets. However, given the number of remaining edx-platform frontends and speed at which they are currently being replatformed, estimates for completion of this process range from one to five years. Thus, in the medium term future, we feel that timeboxed improvements to how edx-platform handles static assets are worthwhile, especially when they address an acute pain point. -Types of asset processing -========================= +In particular, three recent issues have surfaced in Developer Experience Working Group discussions, each with some mitigations involving static assets: + +.. list-table:: + + * - Problem + - Potential solutions + + * - edx-platform Docker images are too large and/or take too long to build. + - Switch from large, legacy tooling packages (such as libsass-python and paver) to industry standard, precompiled ones (like node-sass or dart-sass). Remove unneccessary & slow calls to Django management commands. + + * - edx-platform Docker image layers seem to be rebuilt more often than they should. + - Remove all Python dependencies from the static asset build process, such that changes to Python code or requirements do not always have to result in a static asset rebuild. + + * - In Tutor, using a local copy of edx-platform overwrites the Docker image's pre-installed node_modules and pre-built static assets, requiring developers to reinstall & rebuild in order to get a working platform. + - Parameterize the edx-platform asset build, such that it may search for node_modules outside of edx-platform and genreate assets outside of edx-platform. There are three actions a developer or a deployment pipeline may need to take on edx-platform static assets: * **Build:** Compile, generate, copy, and otherwise process static assets so that they can be used by the Django webserver or collected elsewhere. For many Web applications, all static asset building would be coordinated via Webpack or another NPM-managed tool. Due to the age of edx-platform and its legacy XModule and Comprehensive Themeing systems, though, there are five specific build steps, which generally need to be performed in this order: - #. **Copy npm-installed assets** from node_modules to places where they can be used by certain especially-old edx-platform frontends that do not work with NPM. + #. **Copy npm-installed assets** from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. - #. **Copy XModule Fragments** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. + #. **Copy XModule Fragments** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. This is done for a hard-coded list of XModule-style XBlocks, which are not growing in number; it is *not* a problem for in-repository pure XBlock Fragments or pip-installed XBlock assets, which are ready-to-serve. - #. **Run Webpack** to shim, minify, and bundle JS modules. + #. **Run Webpack** to shim, minify, and bundle JS modules. This requires a call to the npm-installed ``webpack`` binary. #. **Compile Default SCSS** for legacy LMS and CMS frontends into CSS. From bb98788238a3f1767754468328bf09e303a9d984 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 16 Feb 2023 17:14:39 -0500 Subject: [PATCH 09/30] squash: --- docs/decisions/0017-assets-without-python.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index ccbc4f85009d..7f208a40d116 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -52,13 +52,14 @@ New Open edX frontend development has largely moved to React-based micro-fronten - React - Remove as part of full LMS/CMS frontend replatforming -(Note that this table excludes HTML templates. Templates are part of the frontend, but they are dynamically rendered by the Web application and therefore must be handled differently than static assets.) +*Note: this table excludes HTML templates. Templates are part of the frontend, but they are dynamically rendered by the Web application and therefore must be handled differently than static assets.* So, with the exception of XBlock fragments and pip-installed assets, which are very simple for edx-platform to handle, we plan to eventually remove all edx-platform static frontend assets. However, given the number of remaining edx-platform frontends and speed at which they are currently being replatformed, estimates for completion of this process range from one to five years. Thus, in the medium term future, we feel that timeboxed improvements to how edx-platform handles static assets are worthwhile, especially when they address an acute pain point. In particular, three recent issues have surfaced in Developer Experience Working Group discussions, each with some mitigations involving static assets: .. list-table:: + :header-rows: 1 * - Problem - Potential solutions @@ -70,7 +71,7 @@ In particular, three recent issues have surfaced in Developer Experience Working - Remove all Python dependencies from the static asset build process, such that changes to Python code or requirements do not always have to result in a static asset rebuild. * - In Tutor, using a local copy of edx-platform overwrites the Docker image's pre-installed node_modules and pre-built static assets, requiring developers to reinstall & rebuild in order to get a working platform. - - Parameterize the edx-platform asset build, such that it may search for node_modules outside of edx-platform and genreate assets outside of edx-platform. + - Better parameterize the input and output paths edx-platform asset build, such that it may search for node_modules outside of edx-platform and generate assets outside of edx-platform. There are three actions a developer or a deployment pipeline may need to take on edx-platform static assets: From aa282877f53883edd61dbd323ad66990b04c696d Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 16 Feb 2023 22:41:42 -0500 Subject: [PATCH 10/30] squash: lots of adr work --- docs/decisions/0017-assets-without-python.rst | 71 ++++++++++++++----- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index 7f208a40d116..40b8815362b7 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -23,14 +23,6 @@ New Open edX frontend development has largely moved to React-based micro-fronten - Description - Example - Expected direction - * - **XBlock Fragments** - - JS and CSS belonging to the pure XBlocks defined in edx-platform - - library_sourced_block.js - - Keep, or extract to per-XBlock repositories - * - **XModule Fragments** - - JS and SCSS belonging to the older XModule-style XBlocks defined in edx-platform - - ProblemBlock (aka CAPA) assets - - Convert to pure XBlock fragments * - **Legacy LMS Frontends** - JS, SCSS, and other resources powering LMS views that have not yet been replatformed into MFEs - Instructor Dashboard assets @@ -43,26 +35,37 @@ New Open edX frontend development has largely moved to React-based micro-fronten - JS modules, SCSS partials, and other resources, usable by both Legacy LMS and CMS Frontends. This includes a few libraries that have been committed to edx-platform in their entirety. - Legacy cookie policy banner; CodeMirror - Remove as part of full LMS/CMS frontend replatforming - * - **pip-installed Assets** - - Pre-compiled static assets shipped with several Python libraries that we install, including XBlocks. Not committed to edx-platform. - - Django Admin, Swagger, Drag-And-Drop XBlock V2 - - Keep * - **npm-installed Assets** - JS modules and CSS files installed via NPM. Not committed to edx-platform. - React - Remove as part of full LMS/CMS frontend replatforming + * - **XModule Fragments** + - JS and SCSS belonging to the older XModule-style XBlocks defined in edx-platform + - ProblemBlock (aka CAPA) assets + - Convert to pure XBlock fragments + * - **XBlock Fragments** + - JS and CSS belonging to the pure XBlocks defined in edx-platform + - library_sourced_block.js + - Keep and/or extract to pip-installed, per-XBlock repositories + * - **pip-installed Assets** + - Pre-compiled static assets shipped with several Python libraries that we install, including XBlocks. Not committed to edx-platform. + - Django Admin, Swagger, Drag-And-Drop XBlock V2 + - Keep *Note: this table excludes HTML templates. Templates are part of the frontend, but they are dynamically rendered by the Web application and therefore must be handled differently than static assets.* So, with the exception of XBlock fragments and pip-installed assets, which are very simple for edx-platform to handle, we plan to eventually remove all edx-platform static frontend assets. However, given the number of remaining edx-platform frontends and speed at which they are currently being replatformed, estimates for completion of this process range from one to five years. Thus, in the medium term future, we feel that timeboxed improvements to how edx-platform handles static assets are worthwhile, especially when they address an acute pain point. -In particular, three recent issues have surfaced in Developer Experience Working Group discussions, each with some mitigations involving static assets: +Current pain points +=================== + +Three particular issues have surfaced in Developer Experience Working Group discussions recently, each with some mitigations involving static assets: .. list-table:: :header-rows: 1 - * - Problem - - Potential solutions + * - Pain Point + - Potential solution(s) * - edx-platform Docker images are too large and/or take too long to build. - Switch from large, legacy tooling packages (such as libsass-python and paver) to industry standard, precompiled ones (like node-sass or dart-sass). Remove unneccessary & slow calls to Django management commands. @@ -73,9 +76,45 @@ In particular, three recent issues have surfaced in Developer Experience Working * - In Tutor, using a local copy of edx-platform overwrites the Docker image's pre-installed node_modules and pre-built static assets, requiring developers to reinstall & rebuild in order to get a working platform. - Better parameterize the input and output paths edx-platform asset build, such that it may search for node_modules outside of edx-platform and generate assets outside of edx-platform. +All of these potential solutions would involve refactoring or entirely replacing parts of the current asset processing system. + +Decision +******** + +We will rewrite edx-platform's asset processing system. We will aim to: + +* Use well-known, npm-installed frontend tooling wherever possible. +* When bespoke processing is required, use standard POSIX tools like Bash. +* When Django/Python is absolutely required, contain its impact so that the rest of the system remains Python-free. +* Avoid unnecessary indirection or abstraction. For this task, extensibility is a non-goal, and simplicity is a virtue. +* Provide a clear migration path from the old system to the new one. +* Enable the future removal of as much legacy frontend tooling code as possible. + +Consequences +************ + +The three top-level edx-platform asset processing actions are *build*, *collect*, and *watch*. The build action can be further broken down into five stages. Here is how those actions and stages will change: + + +.. list-table:: + :header-rows: 1 + + * - Action/Stage + - Description + - Old implementation + - New implementation + + * - **Build** + - Compile, generate, copy, and otherwise process static assets so that they can be used by the Django webserver or collected elsewhere. For many Web applications, all static asset building would be coordinated via Webpack or another NPM-managed tool. Due to the age of edx-platform and its legacy XModule and Comprehensive Theming systems, though, there are five stages which need to be performed in a particular order. + - ``paver update_assets``: yada + - ``assets/build.sh`` + +TODO +==== + There are three actions a developer or a deployment pipeline may need to take on edx-platform static assets: -* **Build:** Compile, generate, copy, and otherwise process static assets so that they can be used by the Django webserver or collected elsewhere. For many Web applications, all static asset building would be coordinated via Webpack or another NPM-managed tool. Due to the age of edx-platform and its legacy XModule and Comprehensive Themeing systems, though, there are five specific build steps, which generally need to be performed in this order: +* **Build:** : #. **Copy npm-installed assets** from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. From 36f98f96fe8b53f8614376a23c0a7b2bb777930a Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 16 Feb 2023 23:38:00 -0500 Subject: [PATCH 11/30] squash: consequences table --- docs/decisions/0017-assets-without-python.rst | 51 ++++++++++++++++--- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index 40b8815362b7..36abb182b1d8 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -29,16 +29,16 @@ New Open edX frontend development has largely moved to React-based micro-fronten - Replatform & DEPR * - **Legacy CMS Frontends** - JS, SCSS, and other resources powering Studio views that have not yet been replatformed into MFEs - - Course outline editor and unit editor + - Course outline editor and unit editor assets - Replatform & DEPR * - **Shared Frontend Files** - - JS modules, SCSS partials, and other resources, usable by both Legacy LMS and CMS Frontends. This includes a few libraries that have been committed to edx-platform in their entirety. + - JS modules, SCSS partials, and other resources, usable by both Legacy LMS and CMS Frontends. This includes a few vendor libraries that have been committed to edx-platform in their entirety. - Legacy cookie policy banner; CodeMirror - Remove as part of full LMS/CMS frontend replatforming * - **npm-installed Assets** - JS modules and CSS files installed via NPM. Not committed to edx-platform. - - React - - Remove as part of full LMS/CMS frontend replatforming + - React, studio-frontend, paragon + - Uninstall as part of full LMS/CMS frontend replatforming * - **XModule Fragments** - JS and SCSS belonging to the older XModule-style XBlocks defined in edx-platform - ProblemBlock (aka CAPA) assets @@ -81,7 +81,7 @@ All of these potential solutions would involve refactoring or entirely replacing Decision ******** -We will rewrite edx-platform's asset processing system. We will aim to: +We will largely reimplement edx-platform's asset processing system. We will aim to: * Use well-known, npm-installed frontend tooling wherever possible. * When bespoke processing is required, use standard POSIX tools like Bash. @@ -93,7 +93,7 @@ We will rewrite edx-platform's asset processing system. We will aim to: Consequences ************ -The three top-level edx-platform asset processing actions are *build*, *collect*, and *watch*. The build action can be further broken down into five stages. Here is how those actions and stages will change: +The three top-level edx-platform asset processing actions are *build*, *collect*, and *watch*. The build action can be further broken down into five stages. Here is how those actions and stages will be reimplemented: .. list-table:: @@ -106,8 +106,43 @@ The three top-level edx-platform asset processing actions are *build*, *collect* * - **Build** - Compile, generate, copy, and otherwise process static assets so that they can be used by the Django webserver or collected elsewhere. For many Web applications, all static asset building would be coordinated via Webpack or another NPM-managed tool. Due to the age of edx-platform and its legacy XModule and Comprehensive Theming systems, though, there are five stages which need to be performed in a particular order. - - ``paver update_assets``: yada - - ``assets/build.sh`` + - ``paver update_assets --skip-collect``: A Python-defined task that calls out to each build stage. + - ``assets/build.sh``: A Bash script that contains all build stages, its command-line interface inspired by Tutor's ``openedx-assets`` script. The script will be runnable on any POSIX system, including macOS and Ubuntu. The script will be linted for common shell scripting mistakes using `shellcheck `_. + + * - **Build:** Copy from node_modules + - Copy npm-installed assets from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. + - N/A (part of ``paver update_assets``) + - ``assets/build.sh npm``: TODO + + * - **Build:** Copy from XModule + - ** XModule Fragments** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. This is done for a hard-coded list of XModule-style XBlocks, which are not growing in number; it is *not* a problem for in-repository pure XBlock Fragments or pip-installed XBlock assets, which are ready-to-serve. + - ``paver process_xmodule_assets`` and ``xmodule_assets``. The former is a Python wrapper of the latter; the latter is a console script pointing to an application-level Python module. That module inspects attributes from legacy XModule-style XBlock classes in order to determine which static assets to copy and what to name them. + - ``assets/build.sh xmodule``: A Bash implementation of XModule asset copying. The aforementioned attributes will be moved from the XModule-style XBlock classes into a simple static JSON file, which the Bash script will be able to read. + + * - **Build:** Webpack + - Run Webpack in order to to shim, minify, otherwise process, and bundle JS modules. This requires a call to the npm-installed ``webpack`` binary. + - ``paver webpack``: A Python wrapper around a call to webpack. Invokes the ``./manage.py [lms|cms] print_setting`` multiple times in order to determine Django settings, adding which can add 20+ seconds to the build. + - ``assets/build.sh webpack``, a Bash wrapper around a call to webpack. The script will accept parameters for Django settings rather than looking them up. Open edX distributions, such as Tutor, can choose how to supply the Django-setting-dervied parameters in an efficient manner. + + * - **Build:** Default SCSS + - Compile the default SCSS for legacy LMS/CMS frontends into CSS. + - ``paver compile_sass``: TODO + - ``assets/build.sh common``: TODO + + * - **Build:** Theme SCSS + - For each comprehensive theme, compile the theme's SCSS for legacy LMS/CMS frontends into CSS. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. + - ``paver compile_sass``: TODO + - ``assets/build.sh themes``: TODO + + * - **Collect** + - Copy static assets from edx-platform to another location (the ``STATIC_ROOT``) so that they can be efficiently served *without* Django's webserver. This step, by nature, requires Python and Django in order to find and organize the assets, which may come from edx-platform itself or from its many installed Python and NPM packages. This is only done for production environments, where it is usually desirable to serve assets with something efficient like NGINX. + - ``paver update_assets``: TODO + - ``./manage.py lms collectstatic && ./manage.py cms collectstatic``: TODO + + * - **Watch** + - Listen for changes to static assets in the background. When a change occurs, rebuild them automatically, so that the Django webserver picks up the changes. This is only necessary in development environments. A few different sets of assets can be watched: + - ``paver watch_assets``: TODO + - ``assets/build.sh --watch``: TODO TODO ==== From 496db7a3d2b04642885481f08b5a47d337f04ef6 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 16 Feb 2023 23:40:20 -0500 Subject: [PATCH 12/30] squash: number stages --- docs/decisions/0017-assets-without-python.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index 36abb182b1d8..6ac958cfd50e 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -99,37 +99,37 @@ The three top-level edx-platform asset processing actions are *build*, *collect* .. list-table:: :header-rows: 1 - * - Action/Stage + * - Action: Stage - Description - Old implementation - New implementation - * - **Build** + * - **Build:** All Stages - Compile, generate, copy, and otherwise process static assets so that they can be used by the Django webserver or collected elsewhere. For many Web applications, all static asset building would be coordinated via Webpack or another NPM-managed tool. Due to the age of edx-platform and its legacy XModule and Comprehensive Theming systems, though, there are five stages which need to be performed in a particular order. - ``paver update_assets --skip-collect``: A Python-defined task that calls out to each build stage. - ``assets/build.sh``: A Bash script that contains all build stages, its command-line interface inspired by Tutor's ``openedx-assets`` script. The script will be runnable on any POSIX system, including macOS and Ubuntu. The script will be linted for common shell scripting mistakes using `shellcheck `_. - * - **Build:** Copy from node_modules + * - **Build 1/5:** Copy from node_modules - Copy npm-installed assets from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. - N/A (part of ``paver update_assets``) - ``assets/build.sh npm``: TODO - * - **Build:** Copy from XModule + * - **Build 2/5:** Copy from XModule - ** XModule Fragments** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. This is done for a hard-coded list of XModule-style XBlocks, which are not growing in number; it is *not* a problem for in-repository pure XBlock Fragments or pip-installed XBlock assets, which are ready-to-serve. - ``paver process_xmodule_assets`` and ``xmodule_assets``. The former is a Python wrapper of the latter; the latter is a console script pointing to an application-level Python module. That module inspects attributes from legacy XModule-style XBlock classes in order to determine which static assets to copy and what to name them. - ``assets/build.sh xmodule``: A Bash implementation of XModule asset copying. The aforementioned attributes will be moved from the XModule-style XBlock classes into a simple static JSON file, which the Bash script will be able to read. - * - **Build:** Webpack + * - **Build 3/5:** Webpack - Run Webpack in order to to shim, minify, otherwise process, and bundle JS modules. This requires a call to the npm-installed ``webpack`` binary. - ``paver webpack``: A Python wrapper around a call to webpack. Invokes the ``./manage.py [lms|cms] print_setting`` multiple times in order to determine Django settings, adding which can add 20+ seconds to the build. - ``assets/build.sh webpack``, a Bash wrapper around a call to webpack. The script will accept parameters for Django settings rather than looking them up. Open edX distributions, such as Tutor, can choose how to supply the Django-setting-dervied parameters in an efficient manner. - * - **Build:** Default SCSS + * - **Build 4/5:** Default SCSS - Compile the default SCSS for legacy LMS/CMS frontends into CSS. - ``paver compile_sass``: TODO - ``assets/build.sh common``: TODO - * - **Build:** Theme SCSS + * - **Build 5/5:** Theme SCSS - For each comprehensive theme, compile the theme's SCSS for legacy LMS/CMS frontends into CSS. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. - ``paver compile_sass``: TODO - ``assets/build.sh themes``: TODO From 58e271ab627a9db6e6f0a2f8140797474588ef61 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 16 Feb 2023 23:47:03 -0500 Subject: [PATCH 13/30] squash: consequences: 3 columns --- docs/decisions/0017-assets-without-python.rst | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index 6ac958cfd50e..1f87382c4345 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -99,38 +99,31 @@ The three top-level edx-platform asset processing actions are *build*, *collect* .. list-table:: :header-rows: 1 - * - Action: Stage - - Description + * - Description - Old implementation - New implementation - * - **Build:** All Stages - - Compile, generate, copy, and otherwise process static assets so that they can be used by the Django webserver or collected elsewhere. For many Web applications, all static asset building would be coordinated via Webpack or another NPM-managed tool. Due to the age of edx-platform and its legacy XModule and Comprehensive Theming systems, though, there are five stages which need to be performed in a particular order. + * - **Build: All stages.** Compile, generate, copy, and otherwise process static assets so that they can be used by the Django webserver or collected elsewhere. For many Web applications, all static asset building would be coordinated via Webpack or another NPM-managed tool. Due to the age of edx-platform and its legacy XModule and Comprehensive Theming systems, though, there are five stages which need to be performed in a particular order. - ``paver update_assets --skip-collect``: A Python-defined task that calls out to each build stage. - ``assets/build.sh``: A Bash script that contains all build stages, its command-line interface inspired by Tutor's ``openedx-assets`` script. The script will be runnable on any POSIX system, including macOS and Ubuntu. The script will be linted for common shell scripting mistakes using `shellcheck `_. - * - **Build 1/5:** Copy from node_modules - - Copy npm-installed assets from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. + * - **Build 1/5: Copy npm-installed assets** from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. - N/A (part of ``paver update_assets``) - ``assets/build.sh npm``: TODO - * - **Build 2/5:** Copy from XModule - - ** XModule Fragments** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. This is done for a hard-coded list of XModule-style XBlocks, which are not growing in number; it is *not* a problem for in-repository pure XBlock Fragments or pip-installed XBlock assets, which are ready-to-serve. + * - **Build 2/5: Copy XModule framents** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. This is done for a hard-coded list of XModule-style XBlocks, which are not growing in number; it is *not* a problem for in-repository pure XBlock Fragments or pip-installed XBlock assets, which are ready-to-serve. - ``paver process_xmodule_assets`` and ``xmodule_assets``. The former is a Python wrapper of the latter; the latter is a console script pointing to an application-level Python module. That module inspects attributes from legacy XModule-style XBlock classes in order to determine which static assets to copy and what to name them. - ``assets/build.sh xmodule``: A Bash implementation of XModule asset copying. The aforementioned attributes will be moved from the XModule-style XBlock classes into a simple static JSON file, which the Bash script will be able to read. - * - **Build 3/5:** Webpack - - Run Webpack in order to to shim, minify, otherwise process, and bundle JS modules. This requires a call to the npm-installed ``webpack`` binary. + * - **Build 3/5: Run Webpack** in order to to shim, minify, otherwise process, and bundle JS modules. This requires a call to the npm-installed ``webpack`` binary. - ``paver webpack``: A Python wrapper around a call to webpack. Invokes the ``./manage.py [lms|cms] print_setting`` multiple times in order to determine Django settings, adding which can add 20+ seconds to the build. - ``assets/build.sh webpack``, a Bash wrapper around a call to webpack. The script will accept parameters for Django settings rather than looking them up. Open edX distributions, such as Tutor, can choose how to supply the Django-setting-dervied parameters in an efficient manner. - * - **Build 4/5:** Default SCSS - - Compile the default SCSS for legacy LMS/CMS frontends into CSS. + * - **Build 4/5: Compile default SCSS** into CSS for legacy LMS/CMS frontends. - ``paver compile_sass``: TODO - ``assets/build.sh common``: TODO - * - **Build 5/5:** Theme SCSS - - For each comprehensive theme, compile the theme's SCSS for legacy LMS/CMS frontends into CSS. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. + * - **Build 5/5: Compiled themes' SCSS** into CSS for legacy LMS/CMS frontends. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. - ``paver compile_sass``: TODO - ``assets/build.sh themes``: TODO From d2f69f6747ccf09716325dd4714cd886ca5a5206 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 16 Feb 2023 23:50:30 -0500 Subject: [PATCH 14/30] squash: fix table build stage numbering --- docs/decisions/0017-assets-without-python.rst | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index 1f87382c4345..e7ee23ed8985 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -107,33 +107,31 @@ The three top-level edx-platform asset processing actions are *build*, *collect* - ``paver update_assets --skip-collect``: A Python-defined task that calls out to each build stage. - ``assets/build.sh``: A Bash script that contains all build stages, its command-line interface inspired by Tutor's ``openedx-assets`` script. The script will be runnable on any POSIX system, including macOS and Ubuntu. The script will be linted for common shell scripting mistakes using `shellcheck `_. - * - **Build 1/5: Copy npm-installed assets** from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. + * - **Build stage 1: Copy npm-installed assets** from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. - N/A (part of ``paver update_assets``) - ``assets/build.sh npm``: TODO - * - **Build 2/5: Copy XModule framents** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. This is done for a hard-coded list of XModule-style XBlocks, which are not growing in number; it is *not* a problem for in-repository pure XBlock Fragments or pip-installed XBlock assets, which are ready-to-serve. + * - **Build stage 2: Copy XModule framents** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. This is done for a hard-coded list of XModule-style XBlocks, which are not growing in number; it is *not* a problem for in-repository pure XBlock Fragments or pip-installed XBlock assets, which are ready-to-serve. - ``paver process_xmodule_assets`` and ``xmodule_assets``. The former is a Python wrapper of the latter; the latter is a console script pointing to an application-level Python module. That module inspects attributes from legacy XModule-style XBlock classes in order to determine which static assets to copy and what to name them. - ``assets/build.sh xmodule``: A Bash implementation of XModule asset copying. The aforementioned attributes will be moved from the XModule-style XBlock classes into a simple static JSON file, which the Bash script will be able to read. - * - **Build 3/5: Run Webpack** in order to to shim, minify, otherwise process, and bundle JS modules. This requires a call to the npm-installed ``webpack`` binary. + * - **Build stage 3: Run Webpack** in order to to shim, minify, otherwise process, and bundle JS modules. This requires a call to the npm-installed ``webpack`` binary. - ``paver webpack``: A Python wrapper around a call to webpack. Invokes the ``./manage.py [lms|cms] print_setting`` multiple times in order to determine Django settings, adding which can add 20+ seconds to the build. - ``assets/build.sh webpack``, a Bash wrapper around a call to webpack. The script will accept parameters for Django settings rather than looking them up. Open edX distributions, such as Tutor, can choose how to supply the Django-setting-dervied parameters in an efficient manner. - * - **Build 4/5: Compile default SCSS** into CSS for legacy LMS/CMS frontends. + * - **Build stage 4: Compile default SCSS** into CSS for legacy LMS/CMS frontends. - ``paver compile_sass``: TODO - ``assets/build.sh common``: TODO - * - **Build 5/5: Compiled themes' SCSS** into CSS for legacy LMS/CMS frontends. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. + * - **Build stage 5: Compiled themes' SCSS** into CSS for legacy LMS/CMS frontends. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. - ``paver compile_sass``: TODO - ``assets/build.sh themes``: TODO - * - **Collect** - - Copy static assets from edx-platform to another location (the ``STATIC_ROOT``) so that they can be efficiently served *without* Django's webserver. This step, by nature, requires Python and Django in order to find and organize the assets, which may come from edx-platform itself or from its many installed Python and NPM packages. This is only done for production environments, where it is usually desirable to serve assets with something efficient like NGINX. + * - **Collect** the built static assets from edx-platform to another location (the ``STATIC_ROOT``) so that they can be efficiently served *without* Django's webserver. This step, by nature, requires Python and Django in order to find and organize the assets, which may come from edx-platform itself or from its many installed Python and NPM packages. This is only done for production environments, where it is usually desirable to serve assets with something efficient like NGINX. - ``paver update_assets``: TODO - ``./manage.py lms collectstatic && ./manage.py cms collectstatic``: TODO - * - **Watch** - - Listen for changes to static assets in the background. When a change occurs, rebuild them automatically, so that the Django webserver picks up the changes. This is only necessary in development environments. A few different sets of assets can be watched: + * - **Watch** static assets for changes in the background. When a change occurs, rebuild them automatically, so that the Django webserver picks up the changes. This is only necessary in development environments. A few different sets of assets can be watched: - ``paver watch_assets``: TODO - ``assets/build.sh --watch``: TODO From f0ab4759e396eeb1531477a63df863515e9ff464 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 16 Feb 2023 23:58:04 -0500 Subject: [PATCH 15/30] squash: fix watch row, etc --- docs/decisions/0017-assets-without-python.rst | 111 ++++-------------- 1 file changed, 21 insertions(+), 90 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index e7ee23ed8985..1c56c217f139 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -78,6 +78,20 @@ Three particular issues have surfaced in Developer Experience Working Group disc All of these potential solutions would involve refactoring or entirely replacing parts of the current asset processing system. +WIP: Move these links +===================== + +.. _paver: https://github.com/openedx/tutor/tree/open-release/olive.1/pavelib +.. _openedx-assets: https://github.com/overhangio/tutor/blob/v15.0.0/tutor/templates/build/openedx/bin/openedx-assets. + +Updating the asset build pipeline will be necessary for several current and upcoming efforts, including: + +* `Finish upgrading frontend frameworks `_ +* `Move node_modules outside of edx-platform in Tutor's openedx image `_ +* `Move static assets outside of edx-platform in Tutor's openedx image `_ + +This has caused us to consider the value of updating the asset pipeline in place, versus rewriting and simplying it first. + Decision ******** @@ -120,7 +134,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* - ``assets/build.sh webpack``, a Bash wrapper around a call to webpack. The script will accept parameters for Django settings rather than looking them up. Open edX distributions, such as Tutor, can choose how to supply the Django-setting-dervied parameters in an efficient manner. * - **Build stage 4: Compile default SCSS** into CSS for legacy LMS/CMS frontends. - - ``paver compile_sass``: TODO + - ``paver compile_sass``: TODO. Mention libsass. - ``assets/build.sh common``: TODO * - **Build stage 5: Compiled themes' SCSS** into CSS for legacy LMS/CMS frontends. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. @@ -131,103 +145,20 @@ The three top-level edx-platform asset processing actions are *build*, *collect* - ``paver update_assets``: TODO - ``./manage.py lms collectstatic && ./manage.py cms collectstatic``: TODO - * - **Watch** static assets for changes in the background. When a change occurs, rebuild them automatically, so that the Django webserver picks up the changes. This is only necessary in development environments. A few different sets of assets can be watched: + * - **Watch** static assets for changes in the background. When a change occurs, rebuild them automatically, so that the Django webserver picks up the changes. This is only necessary in development environments. A few different sets of assets may be watched: XModule assets, Webpack assets, default SCSS, and theme SCSS. - ``paver watch_assets``: TODO - - ``assets/build.sh --watch``: TODO - -TODO -==== - -There are three actions a developer or a deployment pipeline may need to take on edx-platform static assets: - -* **Build:** : - - #. **Copy npm-installed assets** from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. - - #. **Copy XModule Fragments** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. This is done for a hard-coded list of XModule-style XBlocks, which are not growing in number; it is *not* a problem for in-repository pure XBlock Fragments or pip-installed XBlock assets, which are ready-to-serve. - - #. **Run Webpack** to shim, minify, and bundle JS modules. This requires a call to the npm-installed ``webpack`` binary. - - #. **Compile Default SCSS** for legacy LMS and CMS frontends into CSS. - - #. **Compile Theme SCSS** for legacy LMS and CMS frontends into CSS. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. - -* **Collect:** Copy static assets from edx-platform to another location (the ``STATIC_ROOT``) so that they can be efficiently served *without* Django's webserver. This step, by nature, requires Python and Django in order to find and organize the assets, which may come from edx-platform itself or from its many installed Python and NPM packages. This is only done for production environments, where it is usually desirable to serve assets with something efficient like NGINX. - -* **Watch:** Listen for changes to static assets in the background. When a change occurs, rebuild them automatically, so that the Django webserver picks up the changes. This is only necessary in development environments. A few different sets of assets can be watched: - - * XModule assets. Upon change, these should be re-copied, which should trigger a Webpack re-run and a defualt SCSS recompilation. - - * JavaScript modules. Upon change, a Webpack re-run should be triggered. - - * Default SCSS. Upon change, it should be re-compiled, as should theme SCSS. - - * Theme SCSS. Upon change, it should be re-compiled. + - ``assets/build.sh --watch ``, where ```_ -* `Move node_modules outside of edx-platform in Tutor's openedx image `_ -* `Move static assets outside of edx-platform in Tutor's openedx image `_ - -This has caused us to consider the value of updating the asset pipeline in place, versus rewriting and simplying it first. - -Decision -******** +Notes on Tutor +============== TODO -Rationale: - - * Other parts of pavelib have already been reimplemented, like Python - unit tests. We're following that trend. - * The Python logic in pavelib is harder to understand than simple - shell scripts. - * pavelib has dependencies (Python, paver, edx-platform, other libs) - which means that any pavelib scripts must be executed later in - the edx-platform build process than we might want them to. For - example, in a Dockerfile, it might be more performant to process - npm assets *before* installing Python, but as long as we are still - using pavelib, that is not an option. - * The benefits of paver have been eclipsed by other tools, like - Docker (for requisite management) and Click (for CLI building). - * In the next couple commits, we make improvements to - process-npm-assets.sh. These improvements would have been possible - in the pavelib implementation, but would have been more complicated. -... - -Consequences -************ +Deprecation of the old asset processing system +============================================== TODO -... - Alternatives Considered *********************** From 4538339148bab0d5bc0ea2800e9d2a4224bd46be Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 16 Feb 2023 23:58:53 -0500 Subject: [PATCH 16/30] squash: --- docs/decisions/0017-assets-without-python.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index 1c56c217f139..d579a1f43791 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -84,13 +84,10 @@ WIP: Move these links .. _paver: https://github.com/openedx/tutor/tree/open-release/olive.1/pavelib .. _openedx-assets: https://github.com/overhangio/tutor/blob/v15.0.0/tutor/templates/build/openedx/bin/openedx-assets. -Updating the asset build pipeline will be necessary for several current and upcoming efforts, including: - * `Finish upgrading frontend frameworks `_ * `Move node_modules outside of edx-platform in Tutor's openedx image `_ * `Move static assets outside of edx-platform in Tutor's openedx image `_ -This has caused us to consider the value of updating the asset pipeline in place, versus rewriting and simplying it first. Decision ******** From 7fa25bb8c33210e050da26e16de8f45590fed33e Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 17 Feb 2023 08:45:13 -0500 Subject: [PATCH 17/30] squash: newline & bullets in cell test --- docs/decisions/0017-assets-without-python.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index d579a1f43791..261fe18c846b 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -68,7 +68,8 @@ Three particular issues have surfaced in Developer Experience Working Group disc - Potential solution(s) * - edx-platform Docker images are too large and/or take too long to build. - - Switch from large, legacy tooling packages (such as libsass-python and paver) to industry standard, precompiled ones (like node-sass or dart-sass). Remove unneccessary & slow calls to Django management commands. + - + Switch from large, legacy tooling packages (such as libsass-python and paver) to industry standard, precompiled ones (like node-sass or dart-sass). + + Remove unneccessary & slow calls to Django management commands. * - edx-platform Docker image layers seem to be rebuilt more often than they should. - Remove all Python dependencies from the static asset build process, such that changes to Python code or requirements do not always have to result in a static asset rebuild. @@ -115,7 +116,9 @@ The three top-level edx-platform asset processing actions are *build*, *collect* - New implementation * - **Build: All stages.** Compile, generate, copy, and otherwise process static assets so that they can be used by the Django webserver or collected elsewhere. For many Web applications, all static asset building would be coordinated via Webpack or another NPM-managed tool. Due to the age of edx-platform and its legacy XModule and Comprehensive Theming systems, though, there are five stages which need to be performed in a particular order. - - ``paver update_assets --skip-collect``: A Python-defined task that calls out to each build stage. + - ``paver update_assets --skip-collect`` + + A Python-defined task that calls out to each build stage. - ``assets/build.sh``: A Bash script that contains all build stages, its command-line interface inspired by Tutor's ``openedx-assets`` script. The script will be runnable on any POSIX system, including macOS and Ubuntu. The script will be linted for common shell scripting mistakes using `shellcheck `_. * - **Build stage 1: Copy npm-installed assets** from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. From 187013b04c37d126f5bdb1b122cdef39ecb700b7 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 17 Feb 2023 08:47:39 -0500 Subject: [PATCH 18/30] temp: remove workflows --- .../add-depr-ticket-to-depr-board.yml | 19 -- .github/workflows/check-for-tutorial-prs.yml | 35 --- .github/workflows/ci-static-analysis.yml | 43 --- .github/workflows/commitlint.yml | 10 - .../workflows/docker-compose.yml.mysqldbdump | 23 -- .github/workflows/docker-publish.yml | 22 -- .github/workflows/docs-build-check.yml | 50 ---- .github/workflows/init/01.sql | 3 - .github/workflows/js-tests.yml | 85 ------ .github/workflows/lint-imports.yml | 52 ---- .github/workflows/lockfileversion-check.yml | 13 - .github/workflows/migrations-check-mysql8.yml | 80 ----- .github/workflows/migrations-check.yml | 92 ------ .../workflows/pr-automerge-open-release.yml | 24 -- .github/workflows/publish-ci-docker-image.yml | 35 --- .github/workflows/pylint-checks.yml | 85 ------ .github/workflows/quality-checks.yml | 82 ------ .github/workflows/static-assets-check.yml | 72 ----- .github/workflows/unit-test-shards.json | 277 ------------------ .github/workflows/unit-tests-gh-hosted.yml | 121 -------- .github/workflows/unit-tests.yml | 153 ---------- .../workflows/upgrade-python-requirements.yml | 25 -- .github/workflows/verify-dunder-init.yml | 26 -- .../workflows/verify-gha-unit-tests-count.yml | 23 -- 24 files changed, 1450 deletions(-) delete mode 100644 .github/workflows/add-depr-ticket-to-depr-board.yml delete mode 100644 .github/workflows/check-for-tutorial-prs.yml delete mode 100644 .github/workflows/ci-static-analysis.yml delete mode 100644 .github/workflows/commitlint.yml delete mode 100644 .github/workflows/docker-compose.yml.mysqldbdump delete mode 100644 .github/workflows/docker-publish.yml delete mode 100644 .github/workflows/docs-build-check.yml delete mode 100644 .github/workflows/init/01.sql delete mode 100644 .github/workflows/js-tests.yml delete mode 100644 .github/workflows/lint-imports.yml delete mode 100644 .github/workflows/lockfileversion-check.yml delete mode 100644 .github/workflows/migrations-check-mysql8.yml delete mode 100644 .github/workflows/migrations-check.yml delete mode 100644 .github/workflows/pr-automerge-open-release.yml delete mode 100644 .github/workflows/publish-ci-docker-image.yml delete mode 100644 .github/workflows/pylint-checks.yml delete mode 100644 .github/workflows/quality-checks.yml delete mode 100644 .github/workflows/static-assets-check.yml delete mode 100644 .github/workflows/unit-test-shards.json delete mode 100644 .github/workflows/unit-tests-gh-hosted.yml delete mode 100644 .github/workflows/unit-tests.yml delete mode 100644 .github/workflows/upgrade-python-requirements.yml delete mode 100644 .github/workflows/verify-dunder-init.yml delete mode 100644 .github/workflows/verify-gha-unit-tests-count.yml diff --git a/.github/workflows/add-depr-ticket-to-depr-board.yml b/.github/workflows/add-depr-ticket-to-depr-board.yml deleted file mode 100644 index 73ca4c5c6e87..000000000000 --- a/.github/workflows/add-depr-ticket-to-depr-board.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Run the workflow that adds new tickets that are either: -# - labelled "DEPR" -# - title starts with "[DEPR]" -# - body starts with "Proposal Date" (this is the first template field) -# to the org-wide DEPR project board - -name: Add newly created DEPR issues to the DEPR project board - -on: - issues: - types: [opened] - -jobs: - routeissue: - uses: openedx/.github/.github/workflows/add-depr-ticket-to-depr-board.yml@master - secrets: - GITHUB_APP_ID: ${{ secrets.GRAPHQL_AUTH_APP_ID }} - GITHUB_APP_PRIVATE_KEY: ${{ secrets.GRAPHQL_AUTH_APP_PEM }} - SLACK_BOT_TOKEN: ${{ secrets.SLACK_ISSUE_BOT_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/check-for-tutorial-prs.yml b/.github/workflows/check-for-tutorial-prs.yml deleted file mode 100644 index 6920542ac187..000000000000 --- a/.github/workflows/check-for-tutorial-prs.yml +++ /dev/null @@ -1,35 +0,0 @@ -# This workflow detects PRs that make changes to lms/templates/dashboard.html -# and only lms/templates/dashboard.html. This is the file that users are -# guided through changing in the Open edX tutorial: -# https://docs.openedx.org/en/latest/developers/quickstarts/first_openedx_pr.html#exercise-update-the-learner-dashboard - -# If this is the only file changed in the PR, we comment on the PR congratulating -# the user and letting others know that this is not a community PR in need of -# review. CODEOWNERS will tag a triaging team to provide reviews & ultimately -# close the PR. - -name: Check for Tutorial PR -description: Welcome contributors making their first PR from the tutorial -on: - pull_request: - types: [opened] - paths: - - 'lms/templates/dashboard.html' - -jobs: - # Provide helpful bot comment - comment: - runs-on: ubuntu-latest - name: provide helpful bot comment - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Comment PR - uses: thollander/actions-comment-pull-request@v1 - with: - message: | - Thank you for your pull request! Congratulations on completing the Open edX tutorial! A team member will be by to take a look shortly. - To those watching community pull requests: No need to worry about this one, a tCRIL team member will be taking care of it. - For this PR's author: If this is a PR that is NOT coming from the Open edX tutorial, please comment and let us know to disregard this message. - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml deleted file mode 100644 index 94b6b30600a8..000000000000 --- a/.github/workflows/ci-static-analysis.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Static analysis - -on: pull_request - -jobs: - tests: - name: Static analysis - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: ['3.8'] - os: ['ubuntu-20.04'] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system requirements - run: sudo apt update && sudo apt install -y libxmlsec1-dev - - - name: Install pip - run: python -m pip install -r requirements/pip.txt - - - name: Get pip cache dir - id: pip-cache-dir - run: echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install python dependencies - run: make dev-requirements - - - name: Static code analysis - run: make check-types diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml deleted file mode 100644 index fec11d6c259b..000000000000 --- a/.github/workflows/commitlint.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Run commitlint on the commit messages in a pull request. - -name: Lint Commit Messages - -on: - - pull_request - -jobs: - commitlint: - uses: openedx/.github/.github/workflows/commitlint.yml@master diff --git a/.github/workflows/docker-compose.yml.mysqldbdump b/.github/workflows/docker-compose.yml.mysqldbdump deleted file mode 100644 index 0853d250ff40..000000000000 --- a/.github/workflows/docker-compose.yml.mysqldbdump +++ /dev/null @@ -1,23 +0,0 @@ -version: '3' -services: - mysql: - image: mysql:5.7 - container_name: edx.devstack.mysql57 - ports: - - '3306:3306' - environment: - MYSQL_ROOT_PASSWORD: "" - MYSQL_ALLOW_EMPTY_PASSWORD: "yes" - volumes: - - ./init:/docker-entrypoint-initdb.d - healthcheck: - test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] - timeout: 20s - retries: 10 - edxapp: - image: edxops/edxapp:latest - command: bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db' - volumes: - - ../../:/edx/app/edxapp/edx-platform - depends_on: - - mysql diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml deleted file mode 100644 index e7d9e0c1fa8e..000000000000 --- a/.github/workflows/docker-publish.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Push Docker Images - -on: - push: - branches: - - master -jobs: - # Push image to GitHub Packages. - # See also https://docs.docker.com/docker-hub/builds/ - push: - runs-on: ubuntu-latest - if: github.event_name == 'push' - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Build and Push docker image - env: - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - run : make docker_push diff --git a/.github/workflows/docs-build-check.yml b/.github/workflows/docs-build-check.yml deleted file mode 100644 index 10dd000ce0e2..000000000000 --- a/.github/workflows/docs-build-check.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Docs build - -on: - pull_request: - push: - branches: - - master - -jobs: - tests: - name: Docs build - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: ['3.8'] - os: ['ubuntu-20.04'] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system requirements - run: sudo apt update && sudo apt install -y libxmlsec1-dev - - - name: Install pip - run: python -m pip install -r requirements/pip.txt - - - name: Get pip cache dir - id: pip-cache-dir - run: echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install python dependencies - run: make dev-requirements - - - name: Install docs requirements - run: pip install -r requirements/edx/doc.txt - - - name: Docs build - run: make docs diff --git a/.github/workflows/init/01.sql b/.github/workflows/init/01.sql deleted file mode 100644 index 93d3a107e35e..000000000000 --- a/.github/workflows/init/01.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE DATABASE IF NOT EXISTS `edxapp`; -CREATE DATABASE IF NOT EXISTS `edxapp_csmh`; -GRANT ALL PRIVILEGES ON *.* TO 'edxapp001'@'%' IDENTIFIED BY 'password'; diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml deleted file mode 100644 index 743b5286d87e..000000000000 --- a/.github/workflows/js-tests.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: Javascript tests - -on: - pull_request: - branches: - - master - push: - branches: - - master - -jobs: - run_tests: - name: JS - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ ubuntu-20.04 ] - node-version: [ 16 ] - python-version: [ 3.8 ] - - steps: - - - uses: actions/checkout@v2 - - name: Fetch master to compare coverage - run: git fetch --depth=1 origin master - - - name: Setup Node - uses: actions/setup-node@v2 - with: - node-version: ${{ matrix.node-version }} - - - name: Setup npm - run: npm i -g npm@8.5.x - - - name: Install Firefox 61.0 - run: | - sudo apt-get purge firefox - wget "https://ftp.mozilla.org/pub/firefox/releases/61.0/linux-x86_64/en-US/firefox-61.0.tar.bz2" - tar -xjf firefox-61.0.tar.bz2 - sudo mv firefox /opt/firefox - sudo ln -s /opt/firefox/firefox /usr/bin/firefox - - - name: Install Required System Packages - run: sudo apt-get update && sudo apt-get install libxmlsec1-dev ubuntu-restricted-extras xvfb - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/base.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install Required Python Dependencies - run: | - make base-requirements - - - uses: c-hive/gha-npm-cache@v1 - - name: Run JS Tests - env: - TEST_SUITE: js-unit - SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh - run: | - npm install -g jest - xvfb-run --auto-servernum ./scripts/all-tests.sh - - - name: Save Job Artifacts - uses: actions/upload-artifact@v2 - with: - name: Build-Artifacts - path: | - reports/**/* - test_root/log/*.png - test_root/log/*.log - **/TEST-*.xml diff --git a/.github/workflows/lint-imports.yml b/.github/workflows/lint-imports.yml deleted file mode 100644 index 63caae452f3a..000000000000 --- a/.github/workflows/lint-imports.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Lint Python Imports - -on: - pull_request: - push: - branches: - - master - -jobs: - - lint-imports: - name: Lint Python Imports - runs-on: ubuntu-20.04 - - steps: - - name: Check out branch - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - - name: Install system requirements - run: sudo apt update && sudo apt install -y libxmlsec1-dev - - - name: Install pip - run: python -m pip install -r requirements/pip.txt - - - name: Get pip cache dir - id: pip-cache-dir - run: echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install python dependencies - run: pip install -r requirements/edx/development.txt - - # As long there are sub-projects[1] in edx-platform, we analyze each - # project separately here, in order to make import-linting errors easier - # to pinpoint. - # - # [1] https://openedx.atlassian.net/browse/BOM-2579 - - - name: Analyze imports (repo root) - run: make lint-imports diff --git a/.github/workflows/lockfileversion-check.yml b/.github/workflows/lockfileversion-check.yml deleted file mode 100644 index 42312e8cbf2d..000000000000 --- a/.github/workflows/lockfileversion-check.yml +++ /dev/null @@ -1,13 +0,0 @@ -#check package-lock file version - -name: Lockfile Version check - -on: - push: - branches: - - master - pull_request: - -jobs: - version-check: - uses: openedx/.github/.github/workflows/lockfileversion-check.yml@master diff --git a/.github/workflows/migrations-check-mysql8.yml b/.github/workflows/migrations-check-mysql8.yml deleted file mode 100644 index 74c4e6cabd84..000000000000 --- a/.github/workflows/migrations-check-mysql8.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Migrations check on MySql 8.0 - -on: - workflow_dispatch: - pull_request: - push: - branches: - - master - -jobs: - check_migrations: - name: check migrations mysql8 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ ubuntu-20.04 ] - python-version: [ 3.8 ] - - steps: - - name: Checkout repo - uses: actions/checkout@v2 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system Packages - run: | - sudo apt-get update - sudo apt-get install -y libxmlsec1-dev - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Ubuntu and sql Versions - run: | - lsb_release -a - mysql -V - - - name: Install Python dependencies - run: | - make dev-requirements - pip uninstall -y mysqlclient - pip install --no-binary mysqlclient mysqlclient - pip uninstall -y xmlsec - pip install --no-binary xmlsec xmlsec - - - name: Initiate Services - run: | - sudo systemctl start mongod - sudo /etc/init.d/mysql start - - - name: Reset mysql password - run: | - cat <> $GITHUB_PATH - - - name: Run Static Assets Check - env: - LMS_CFG: lms/envs/bok_choy.yml - CMS_CFG: cms/envs/bok_choy.yml - - run: | - paver update_assets lms - paver update_assets cms diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json deleted file mode 100644 index c250b00545c9..000000000000 --- a/.github/workflows/unit-test-shards.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "lms-1": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/badges/", - "lms/djangoapps/branding/", - "lms/djangoapps/bulk_email/", - "lms/djangoapps/bulk_enroll/", - "lms/djangoapps/bulk_user_retirement/", - "lms/djangoapps/ccx/", - "lms/djangoapps/certificates/", - "lms/djangoapps/commerce/" - ] - }, - "lms-2": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/course_api/", - "lms/djangoapps/course_blocks/", - "lms/djangoapps/course_goals/", - "lms/djangoapps/course_home_api/", - "lms/djangoapps/course_wiki/", - "lms/djangoapps/coursewarehistoryextended/", - "lms/djangoapps/debug/" - ] - }, - "lms-3": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/courseware/" - ] - }, - "lms-4": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/discussion/", - "lms/djangoapps/edxnotes/", - "lms/djangoapps/email_marketing/", - "lms/djangoapps/experiments/" - ] - }, - "lms-5": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/gating/", - "lms/djangoapps/grades/", - "lms/djangoapps/instructor/", - "lms/djangoapps/instructor_analytics/" - ] - }, - "lms-6": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/instructor_task/", - "lms/djangoapps/learner_dashboard/", - "lms/djangoapps/learner_home/", - "lms/djangoapps/learner_recommendations/", - "lms/djangoapps/lms_initialization/", - "lms/djangoapps/lms_xblock/", - "lms/djangoapps/lti_provider/", - "lms/djangoapps/mailing/", - "lms/djangoapps/mobile_api/", - "lms/djangoapps/monitoring/", - "lms/djangoapps/ora_staff_grader/", - "lms/djangoapps/program_enrollments/", - "lms/djangoapps/rss_proxy/", - "lms/djangoapps/save_for_later/", - "lms/djangoapps/static_template_view/", - "lms/djangoapps/staticbook/", - "lms/djangoapps/support/", - "lms/djangoapps/survey/", - "lms/djangoapps/teams/", - "lms/djangoapps/tests/", - "lms/djangoapps/user_tours/", - "lms/djangoapps/verify_student/", - "lms/djangoapps/mfe_config_api/", - "lms/envs/", - "lms/lib/", - "lms/tests.py" - ] - }, - "openedx-1": { - "settings": "lms.envs.test", - "paths": [ - "openedx/core/djangoapps/ace_common/", - "openedx/core/djangoapps/cors_csrf/", - "openedx/core/djangoapps/agreements/", - "openedx/core/djangoapps/api_admin/", - "openedx/core/djangoapps/auth_exchange/", - "openedx/core/djangoapps/bookmarks/", - "openedx/core/djangoapps/cache_toolbox/", - "openedx/core/djangoapps/catalog/", - "openedx/core/djangoapps/ccxcon/", - "openedx/core/djangoapps/commerce/", - "openedx/core/djangoapps/common_initialization/", - "openedx/core/djangoapps/common_views/", - "openedx/core/djangoapps/config_model_utils/", - "openedx/core/djangoapps/content/", - "openedx/core/djangoapps/content_libraries/", - "openedx/core/djangoapps/contentserver/", - "openedx/core/djangoapps/cookie_metadata/", - "openedx/core/djangoapps/course_apps/", - "openedx/core/djangoapps/course_date_signals/", - "openedx/core/djangoapps/course_groups/", - "openedx/core/djangoapps/courseware_api/", - "openedx/core/djangoapps/crawlers/", - "openedx/core/djangoapps/credentials/", - "openedx/core/djangoapps/credit/", - "openedx/core/djangoapps/course_live/", - "openedx/core/djangoapps/dark_lang/", - "openedx/core/djangoapps/debug/", - "openedx/core/djangoapps/demographics/", - "openedx/core/djangoapps/discussions/", - "openedx/core/djangoapps/django_comment_common/", - "openedx/core/djangoapps/embargo/", - "openedx/core/djangoapps/enrollments/", - "openedx/core/djangoapps/external_user_ids/" - ] - }, - "openedx-2": { - "settings": "lms.envs.test", - "paths": [ - "openedx/core/djangoapps/geoinfo/", - "openedx/core/djangoapps/header_control/", - "openedx/core/djangoapps/heartbeat/", - "openedx/core/djangoapps/lang_pref/", - "openedx/core/djangoapps/models/", - "openedx/core/djangoapps/monkey_patch/", - "openedx/core/djangoapps/oauth_dispatch/", - "openedx/core/djangoapps/olx_rest_api/", - "openedx/core/djangoapps/password_policy/", - "openedx/core/djangoapps/plugin_api/", - "openedx/core/djangoapps/plugins/", - "openedx/core/djangoapps/profile_images/", - "openedx/core/djangoapps/programs/", - "openedx/core/djangoapps/safe_sessions/", - "openedx/core/djangoapps/schedules/", - "openedx/core/djangoapps/service_status/", - "openedx/core/djangoapps/session_inactivity_timeout/", - "openedx/core/djangoapps/signals/", - "openedx/core/djangoapps/site_configuration/", - "openedx/core/djangoapps/system_wide_roles/", - "openedx/core/djangoapps/theming/", - "openedx/core/djangoapps/user_api/", - "openedx/core/djangoapps/user_authn/", - "openedx/core/djangoapps/util/", - "openedx/core/djangoapps/verified_track_content/", - "openedx/core/djangoapps/video_config/", - "openedx/core/djangoapps/video_pipeline/", - "openedx/core/djangoapps/waffle_utils/", - "openedx/core/djangoapps/xblock/", - "openedx/core/djangoapps/xmodule_django/", - "openedx/core/djangoapps/zendesk_proxy/", - "openedx/core/djangolib/", - "openedx/core/lib/", - "openedx/core/tests/", - "openedx/features/", - "openedx/testing/", - "openedx/tests/" - ] - }, - "openedx-3": { - "settings": "cms.envs.test", - "paths": [ - "openedx/core/djangoapps/ace_common/", - "openedx/core/djangoapps/cors_csrf/", - "openedx/core/djangoapps/agreements/", - "openedx/core/djangoapps/api_admin/", - "openedx/core/djangoapps/auth_exchange/", - "openedx/core/djangoapps/bookmarks/", - "openedx/core/djangoapps/cache_toolbox/", - "openedx/core/djangoapps/catalog/", - "openedx/core/djangoapps/ccxcon/", - "openedx/core/djangoapps/commerce/", - "openedx/core/djangoapps/common_initialization/", - "openedx/core/djangoapps/common_views/", - "openedx/core/djangoapps/config_model_utils/", - "openedx/core/djangoapps/content/", - "openedx/core/djangoapps/content_libraries/", - "openedx/core/djangoapps/contentserver/", - "openedx/core/djangoapps/cookie_metadata/", - "openedx/core/djangoapps/course_apps/", - "openedx/core/djangoapps/course_date_signals/", - "openedx/core/djangoapps/course_groups/", - "openedx/core/djangoapps/courseware_api/", - "openedx/core/djangoapps/crawlers/", - "openedx/core/djangoapps/credentials/", - "openedx/core/djangoapps/credit/", - "openedx/core/djangoapps/dark_lang/", - "openedx/core/djangoapps/debug/", - "openedx/core/djangoapps/demographics/", - "openedx/core/djangoapps/discussions/", - "openedx/core/djangoapps/django_comment_common/", - "openedx/core/djangoapps/embargo/", - "openedx/core/djangoapps/enrollments/", - "openedx/core/djangoapps/external_user_ids/" - ] - }, - "openedx-4": { - "settings": "cms.envs.test", - "paths": [ - "openedx/core/djangoapps/geoinfo/", - "openedx/core/djangoapps/header_control/", - "openedx/core/djangoapps/heartbeat/", - "openedx/core/djangoapps/lang_pref/", - "openedx/core/djangoapps/models/", - "openedx/core/djangoapps/monkey_patch/", - "openedx/core/djangoapps/oauth_dispatch/", - "openedx/core/djangoapps/olx_rest_api/", - "openedx/core/djangoapps/password_policy/", - "openedx/core/djangoapps/plugin_api/", - "openedx/core/djangoapps/plugins/", - "openedx/core/djangoapps/profile_images/", - "openedx/core/djangoapps/programs/", - "openedx/core/djangoapps/safe_sessions/", - "openedx/core/djangoapps/schedules/", - "openedx/core/djangoapps/service_status/", - "openedx/core/djangoapps/session_inactivity_timeout/", - "openedx/core/djangoapps/signals/", - "openedx/core/djangoapps/site_configuration/", - "openedx/core/djangoapps/system_wide_roles/", - "openedx/core/djangoapps/theming/", - "openedx/core/djangoapps/user_api/", - "openedx/core/djangoapps/user_authn/", - "openedx/core/djangoapps/util/", - "openedx/core/djangoapps/verified_track_content/", - "openedx/core/djangoapps/video_config/", - "openedx/core/djangoapps/video_pipeline/", - "openedx/core/djangoapps/waffle_utils/", - "openedx/core/djangoapps/xblock/", - "openedx/core/djangoapps/xmodule_django/", - "openedx/core/djangoapps/zendesk_proxy/", - "openedx/core/lib/", - "openedx/tests/" - ] - }, - "cms-1": { - "settings": "cms.envs.test", - "paths": [ - "cms/djangoapps/api/", - "cms/djangoapps/cms_user_tasks/", - "cms/djangoapps/coursegraph/", - "cms/djangoapps/course_creators/", - "cms/djangoapps/export_course_metadata/", - "cms/djangoapps/maintenance/", - "cms/djangoapps/models/", - "cms/djangoapps/pipeline_js/", - "cms/djangoapps/xblock_config/", - "cms/envs/", - "cms/lib/" - ] - }, - "cms-2": { - "settings": "cms.envs.test", - "paths": [ - "cms/djangoapps/contentstore/" - ] - }, - "common-1": { - "settings": "lms.envs.test", - "paths": [ - "common/djangoapps/" - ] - }, - "common-2": { - "settings": "cms.envs.test", - "paths": [ - "common/djangoapps/" - ] - }, - "xmodule-1": { - "settings": "lms.envs.test", - "paths": [ - "xmodule/" - ] - } -} diff --git a/.github/workflows/unit-tests-gh-hosted.yml b/.github/workflows/unit-tests-gh-hosted.yml deleted file mode 100644 index fc5f9bee6371..000000000000 --- a/.github/workflows/unit-tests-gh-hosted.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: unit-tests-gh-hosted - -on: - pull_request: - push: - branches: - - master - - open-release/lilac.master - -jobs: - run-test: - if: (github.repository != 'openedx/edx-platform' && github.repository != 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == true)) - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - python-version: [ '3.8' ] - django-version: - - "pinned" - shard_name: [ - "lms-1", - "lms-2", - "lms-3", - "lms-4", - "lms-5", - "lms-6", - "openedx-1", - "openedx-2", - "openedx-3", - "openedx-4", - "cms-1", - "cms-2", - "common-1", - "common-2", - "common-3", - ] - name: gh-hosted-python-${{ matrix.python-version }},django-${{ matrix.django-version }},${{ matrix.shard_name }} - steps: - - uses: actions/checkout@v2 - - - name: Install Required System Packages - run: sudo apt-get update && sudo apt-get install libxmlsec1-dev lynx - - - name: Start MongoDB - uses: supercharge/mongodb-github-action@1.7.0 - with: - mongodb-version: 4.4 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install Required Python Dependencies - env: - PIP_SRC: ${{ runner.temp }} - run: | - make test-requirements - if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - pip install "django~=${{ matrix.django-version }}.0" - pip check # fail if this test-reqs/Django combination is broken - fi - - - name: Setup and run tests - uses: ./.github/actions/unit-tests - - collect-and-verify: - if: (github.repository != 'openedx/edx-platform' && github.repository != 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == true)) - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: [ '3.8' ] - django-version: - - "pinned" - steps: - - uses: actions/checkout@v2 - - - name: Install Required System Packages - run: sudo apt-get update && sudo apt-get install libxmlsec1-dev - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install Required Python Dependencies - run: | - make test-requirements - if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - pip install "django~=${{ matrix.django-version }}.0" - pip check # fail if this test-reqs/Django combination is broken - fi - - - name: verify unit tests count - uses: ./.github/actions/verify-tests-count diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml deleted file mode 100644 index 69a03bbeafff..000000000000 --- a/.github/workflows/unit-tests.yml +++ /dev/null @@ -1,153 +0,0 @@ -name: unit-tests - -on: - pull_request: - push: - branches: - - master - -jobs: - run-tests: - name: python-${{ matrix.python-version }},django-${{ matrix.django-version }},${{ matrix.shard_name }} - if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) - runs-on: [ edx-platform-runner ] - strategy: - matrix: - python-version: - - "3.8" - django-version: - - "pinned" - #- "4.0" - shard_name: - - "lms-1" - - "lms-2" - - "lms-3" - - "lms-4" - - "lms-5" - - "lms-6" - - "openedx-1" - - "openedx-2" - - "openedx-3" - - "openedx-4" - - "cms-1" - - "cms-2" - - "common-1" - - "common-2" - - "xmodule-1" - # We expect Django 4.0 to fail, so don't stop when it fails. - continue-on-error: ${{ matrix.django-version == '4.0' }} - - steps: - - name: sync directory owner - run: sudo chown runner:runner -R .* - - - name: checkout repo - uses: actions/checkout@v3 - - - name: start mongod server for tests - run: | - sudo mkdir -p /data/db - sudo chmod -R a+rw /data/db - mongod & - - - name: install requirements - run: | - sudo make test-requirements - if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - sudo pip install "django~=${{ matrix.django-version }}.0" - sudo pip check # fail if this test-reqs/Django combination is broken - fi - - - name: list installed package versions - run: | - sudo pip freeze - - - name: Setup and run tests - uses: ./.github/actions/unit-tests - - - name: Renaming coverage data file - run: | - mv reports/.coverage reports/${{ matrix.shard_name }}.coverage - - - name: Upload coverage - uses: actions/upload-artifact@v3 - with: - name: coverage - path: reports/${{matrix.shard_name}}.coverage - - # This job aggregates test results. It's the required check for branch protection. - # https://github.com/marketplace/actions/alls-green#why - # https://github.com/orgs/community/discussions/33579 - success: - name: Tests successful - if: always() - needs: - - run-tests - runs-on: ubuntu-latest - steps: - - name: Decide whether the needed jobs succeeded or failed - # uses: re-actors/alls-green@v1.2.1 - uses: re-actors/alls-green@13b4244b312e8a314951e03958a2f91519a6a3c9 - with: - jobs: ${{ toJSON(needs) }} - - compile-warnings-report: - runs-on: [ edx-platform-runner ] - needs: [ run-tests ] - steps: - - name: sync directory owner - run: sudo chown runner:runner -R .* - - uses: actions/checkout@v3 - - name: collect pytest warnings files - uses: actions/download-artifact@v2 - with: - name: pytest-warnings-json - path: test_root/log - - - name: display structure of downloaded files - run: ls -la test_root/log - - - name: compile warnings report - run: | - python openedx/core/process_warnings.py --dir-path test_root/log --html-path reports/pytest_warnings/warning_report_all.html - - - name: save warning report - if: success() - uses: actions/upload-artifact@v3 - with: - name: pytest-warning-report-html - path: | - reports/pytest_warnings/warning_report_all.html - - # Combine and upload coverage reports. - coverage: - needs: run-tests - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [ 3.8 ] - steps: - - name: Checkout repo - uses: actions/checkout@v3 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Download all artifacts - uses: actions/download-artifact@v3 - with: - name: coverage - path: reports - - - name: Install Python dependencies - run: | - pip install -r requirements/edx/coverage.txt - - - name: Run coverage - run: | - coverage combine reports/* - coverage report - coverage xml - - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml deleted file mode 100644 index 911c6f0e51da..000000000000 --- a/.github/workflows/upgrade-python-requirements.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Upgrade Requirements - -on: - schedule: - - cron: "0 2 * * 2" - workflow_dispatch: - inputs: - branch: - description: 'Target branch to create requirements PR against' - required: true - default: 'master' -jobs: - call-upgrade-python-requirements-workflow: - with: - branch: ${{ github.event.inputs.branch }} - team_reviewers: "arbi-bom" - email_address: arbi-bom@edx.org - send_success_notification: false - secrets: - requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }} - requirements_bot_github_email: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }} - edx_smtp_username: ${{ secrets.EDX_SMTP_USERNAME }} - edx_smtp_password: ${{ secrets.EDX_SMTP_PASSWORD }} - uses: openedx/.github/.github/workflows/upgrade-python-requirements.yml@master - diff --git a/.github/workflows/verify-dunder-init.yml b/.github/workflows/verify-dunder-init.yml deleted file mode 100644 index aefc0f53b6f5..000000000000 --- a/.github/workflows/verify-dunder-init.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: CI - -on: - pull_request: - branches: - - master - -jobs: - - verify_dunder_init: - - name: Verify __init__.py Files - runs-on: ubuntu-20.04 - - steps: - - - name: Check out branch - uses: actions/checkout@v2 - - - name: Ensure git is installed - run: | - sudo apt-get update && sudo apt-get install git - - - name: Verify __init__.py files exist - run: | - scripts/verify-dunder-init.sh diff --git a/.github/workflows/verify-gha-unit-tests-count.yml b/.github/workflows/verify-gha-unit-tests-count.yml deleted file mode 100644 index c68092942d70..000000000000 --- a/.github/workflows/verify-gha-unit-tests-count.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: verify unit tests count - -on: - pull_request: - push: - branches: - - master - -jobs: - collect-and-verify: - if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) - runs-on: [ edx-platform-runner ] - steps: - - name: sync directory owner - run: sudo chown runner:runner -R .* - - - uses: actions/checkout@v2 - - name: install requirements - run: | - sudo make test-requirements - - - name: verify unit tests count - uses: ./.github/actions/verify-tests-count From ef59d2467c528cb0ede0ee58c780cda80f31c73d Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 17 Feb 2023 08:51:37 -0500 Subject: [PATCH 19/30] squash: table fmt --- docs/decisions/0017-assets-without-python.rst | 70 +++++++++++++++---- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index 261fe18c846b..a93c348038ae 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -68,8 +68,7 @@ Three particular issues have surfaced in Developer Experience Working Group disc - Potential solution(s) * - edx-platform Docker images are too large and/or take too long to build. - - + Switch from large, legacy tooling packages (such as libsass-python and paver) to industry standard, precompiled ones (like node-sass or dart-sass). - + Remove unneccessary & slow calls to Django management commands. + - Switch from large, legacy tooling packages (such as libsass-python and paver) to industry standard, precompiled ones (like node-sass or dart-sass). Remove unneccessary & slow calls to Django management commands. * - edx-platform Docker image layers seem to be rebuilt more often than they should. - Remove all Python dependencies from the static asset build process, such that changes to Python code or requirements do not always have to result in a static asset rebuild. @@ -116,38 +115,79 @@ The three top-level edx-platform asset processing actions are *build*, *collect* - New implementation * - **Build: All stages.** Compile, generate, copy, and otherwise process static assets so that they can be used by the Django webserver or collected elsewhere. For many Web applications, all static asset building would be coordinated via Webpack or another NPM-managed tool. Due to the age of edx-platform and its legacy XModule and Comprehensive Theming systems, though, there are five stages which need to be performed in a particular order. + - ``paver update_assets --skip-collect`` A Python-defined task that calls out to each build stage. - ``assets/build.sh``: A Bash script that contains all build stages, its command-line interface inspired by Tutor's ``openedx-assets`` script. The script will be runnable on any POSIX system, including macOS and Ubuntu. The script will be linted for common shell scripting mistakes using `shellcheck `_. * - **Build stage 1: Copy npm-installed assets** from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. - - N/A (part of ``paver update_assets``) - - ``assets/build.sh npm``: TODO - + + - N/A + + (part of ``paver update_assets``) + + - ``assets/build.sh npm`` + + TODO + * - **Build stage 2: Copy XModule framents** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. This is done for a hard-coded list of XModule-style XBlocks, which are not growing in number; it is *not* a problem for in-repository pure XBlock Fragments or pip-installed XBlock assets, which are ready-to-serve. - - ``paver process_xmodule_assets`` and ``xmodule_assets``. The former is a Python wrapper of the latter; the latter is a console script pointing to an application-level Python module. That module inspects attributes from legacy XModule-style XBlock classes in order to determine which static assets to copy and what to name them. - - ``assets/build.sh xmodule``: A Bash implementation of XModule asset copying. The aforementioned attributes will be moved from the XModule-style XBlock classes into a simple static JSON file, which the Bash script will be able to read. + + - ``paver process_xmodule_assets``, or + ``xmodule_assets`` + + The former is a Python wrapper of the latter; the latter is a console script pointing to an application-level Python module. That module inspects attributes from legacy XModule-style XBlock classes in order to determine which static assets to copy and what to name them. + + - ``assets/build.sh xmodule`` + + A Bash implementation of XModule asset copying. The aforementioned attributes will be moved from the XModule-style XBlock classes into a simple static JSON file, which the Bash script will be able to read. * - **Build stage 3: Run Webpack** in order to to shim, minify, otherwise process, and bundle JS modules. This requires a call to the npm-installed ``webpack`` binary. + - ``paver webpack``: A Python wrapper around a call to webpack. Invokes the ``./manage.py [lms|cms] print_setting`` multiple times in order to determine Django settings, adding which can add 20+ seconds to the build. + - ``assets/build.sh webpack``, a Bash wrapper around a call to webpack. The script will accept parameters for Django settings rather than looking them up. Open edX distributions, such as Tutor, can choose how to supply the Django-setting-dervied parameters in an efficient manner. * - **Build stage 4: Compile default SCSS** into CSS for legacy LMS/CMS frontends. - - ``paver compile_sass``: TODO. Mention libsass. - - ``assets/build.sh common``: TODO + + - ``paver compile_sass`` + + TODO. Mention libsass. + + - ``assets/build.sh common`` + + TODO * - **Build stage 5: Compiled themes' SCSS** into CSS for legacy LMS/CMS frontends. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. - - ``paver compile_sass``: TODO - - ``assets/build.sh themes``: TODO + + - ``paver compile_sass`` + + TODO + + - ``assets/build.sh themes`` + + TODO * - **Collect** the built static assets from edx-platform to another location (the ``STATIC_ROOT``) so that they can be efficiently served *without* Django's webserver. This step, by nature, requires Python and Django in order to find and organize the assets, which may come from edx-platform itself or from its many installed Python and NPM packages. This is only done for production environments, where it is usually desirable to serve assets with something efficient like NGINX. - - ``paver update_assets``: TODO - - ``./manage.py lms collectstatic && ./manage.py cms collectstatic``: TODO + + - ``paver update_assets`` + + TODO + + - ``./manage.py lms collectstatic && ./manage.py cms collectstatic`` + + TODO * - **Watch** static assets for changes in the background. When a change occurs, rebuild them automatically, so that the Django webserver picks up the changes. This is only necessary in development environments. A few different sets of assets may be watched: XModule assets, Webpack assets, default SCSS, and theme SCSS. - - ``paver watch_assets``: TODO - - ``assets/build.sh --watch ``, where ```` + where `` Date: Fri, 17 Feb 2023 08:56:17 -0500 Subject: [PATCH 20/30] squash: --- docs/decisions/0017-assets-without-python.rst | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index a93c348038ae..06fedf631ede 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -119,13 +119,16 @@ The three top-level edx-platform asset processing actions are *build*, *collect* - ``paver update_assets --skip-collect`` A Python-defined task that calls out to each build stage. - - ``assets/build.sh``: A Bash script that contains all build stages, its command-line interface inspired by Tutor's ``openedx-assets`` script. The script will be runnable on any POSIX system, including macOS and Ubuntu. The script will be linted for common shell scripting mistakes using `shellcheck `_. + + - ``assets/build.sh`` + + A Bash script that contains all build stages, its command-line interface inspired by Tutor's ``openedx-assets`` script. The script will be runnable on any POSIX system, including macOS and Ubuntu. The script will be linted for common shell scripting mistakes using `shellcheck `_. * - **Build stage 1: Copy npm-installed assets** from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. - - N/A + - ``paver update_assets --skip-collect`` - (part of ``paver update_assets``) + This stage is implemented in Python within update_assets. There is not standalone command for it. - ``assets/build.sh npm`` @@ -134,9 +137,9 @@ The three top-level edx-platform asset processing actions are *build*, *collect* * - **Build stage 2: Copy XModule framents** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. This is done for a hard-coded list of XModule-style XBlocks, which are not growing in number; it is *not* a problem for in-repository pure XBlock Fragments or pip-installed XBlock assets, which are ready-to-serve. - ``paver process_xmodule_assets``, or - ``xmodule_assets`` + ``xmodule_assets`` - The former is a Python wrapper of the latter; the latter is a console script pointing to an application-level Python module. That module inspects attributes from legacy XModule-style XBlock classes in order to determine which static assets to copy and what to name them. + The former is a Python wrapper of the latter; the latter is a console script pointing to an application-level Python module. That module inspects attributes from legacy XModule-style XBlock classes in order to determine which static assets to copy and what to name them. - ``assets/build.sh xmodule`` @@ -144,9 +147,13 @@ The three top-level edx-platform asset processing actions are *build*, *collect* * - **Build stage 3: Run Webpack** in order to to shim, minify, otherwise process, and bundle JS modules. This requires a call to the npm-installed ``webpack`` binary. - - ``paver webpack``: A Python wrapper around a call to webpack. Invokes the ``./manage.py [lms|cms] print_setting`` multiple times in order to determine Django settings, adding which can add 20+ seconds to the build. + - ``paver webpack`` + + A Python wrapper around a call to webpack. Invokes the ``./manage.py [lms|cms] print_setting`` multiple times in order to determine Django settings, adding which can add 20+ seconds to the build. + + - ``assets/build.sh webpack`` - - ``assets/build.sh webpack``, a Bash wrapper around a call to webpack. The script will accept parameters for Django settings rather than looking them up. Open edX distributions, such as Tutor, can choose how to supply the Django-setting-dervied parameters in an efficient manner. + A Bash wrapper around a call to webpack. The script will accept parameters for Django settings rather than looking them up. Open edX distributions, such as Tutor, can choose how to supply the Django-setting-dervied parameters in an efficient manner. * - **Build stage 4: Compile default SCSS** into CSS for legacy LMS/CMS frontends. @@ -185,9 +192,9 @@ The three top-level edx-platform asset processing actions are *build*, *collect* TODO - ``assets/build.sh --watch `` - where ```` if one of the build stages described above - TODO. + TODO. Notes on Tutor ============== From 8fcefe08b783efb69b7962bbd1a219e7ed0465d6 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 17 Feb 2023 09:05:57 -0500 Subject: [PATCH 21/30] squash: impove implementation cells --- docs/decisions/0017-assets-without-python.rst | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index 06fedf631ede..13d91c05a62c 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -122,38 +122,41 @@ The three top-level edx-platform asset processing actions are *build*, *collect* - ``assets/build.sh`` - A Bash script that contains all build stages, its command-line interface inspired by Tutor's ``openedx-assets`` script. The script will be runnable on any POSIX system, including macOS and Ubuntu. The script will be linted for common shell scripting mistakes using `shellcheck `_. + A Bash script that contains all build stages, with subcommands available for running each stage separately. Its command-line interface inspired by Tutor's ``openedx-assets`` script. The script will be runnable on any POSIX system, including macOS and Ubuntu and it will linted for common shell scripting mistakes using `shellcheck `_. * - **Build stage 1: Copy npm-installed assets** from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. - ``paver update_assets --skip-collect`` - This stage is implemented in Python within update_assets. There is not standalone command for it. + Implemented in Python within update_assets. There is not standalone command for it. - ``assets/build.sh npm`` - TODO + Pure Bash reimplementation. * - **Build stage 2: Copy XModule framents** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. This is done for a hard-coded list of XModule-style XBlocks, which are not growing in number; it is *not* a problem for in-repository pure XBlock Fragments or pip-installed XBlock assets, which are ready-to-serve. - ``paver process_xmodule_assets``, or ``xmodule_assets`` - The former is a Python wrapper of the latter; the latter is a console script pointing to an application-level Python module. That module inspects attributes from legacy XModule-style XBlock classes in order to determine which static assets to copy and what to name them. + Equivalent paver task and console script, both pointing at to an application-level Python module. That module inspects attributes from legacy XModule-style XBlock classes in order to determine which static assets to copy and what to name them. - ``assets/build.sh xmodule`` + A Bash implementation of XModule asset copying. The aforementioned attributes will be moved from the XModule-style XBlock classes into a simple static JSON file, which the Bash script will be able to read. + + (The initial implementation of build.sh may just point at ``xmodule_assets``). * - **Build stage 3: Run Webpack** in order to to shim, minify, otherwise process, and bundle JS modules. This requires a call to the npm-installed ``webpack`` binary. - ``paver webpack`` - A Python wrapper around a call to webpack. Invokes the ``./manage.py [lms|cms] print_setting`` multiple times in order to determine Django settings, adding which can add 20+ seconds to the build. + Python wrapper around a call to webpack. Invokes the ``./manage.py [lms|cms] print_setting`` multiple times in order to determine Django settings, adding which can add 20+ seconds to the build. - ``assets/build.sh webpack`` - A Bash wrapper around a call to webpack. The script will accept parameters for Django settings rather than looking them up. Open edX distributions, such as Tutor, can choose how to supply the Django-setting-dervied parameters in an efficient manner. + Bash wrapper around a call to webpack. The script will accept parameters for Django settings rather than looking them up. Open edX distributions, such as Tutor, can choose how to supply the Django-setting-dervied parameters in an efficient manner. * - **Build stage 4: Compile default SCSS** into CSS for legacy LMS/CMS frontends. @@ -192,6 +195,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* TODO - ``assets/build.sh --watch `` + where ```` if one of the build stages described above TODO. From 1a74c63aefb7913989f3fd65a88387354910d78d Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 17 Feb 2023 09:08:24 -0500 Subject: [PATCH 22/30] squash: build stage bullets --- docs/decisions/0017-assets-without-python.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index 13d91c05a62c..636bacba0315 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -124,7 +124,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* A Bash script that contains all build stages, with subcommands available for running each stage separately. Its command-line interface inspired by Tutor's ``openedx-assets`` script. The script will be runnable on any POSIX system, including macOS and Ubuntu and it will linted for common shell scripting mistakes using `shellcheck `_. - * - **Build stage 1: Copy npm-installed assets** from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. + * - + **Build stage 1: Copy npm-installed assets** from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. - ``paver update_assets --skip-collect`` @@ -134,7 +134,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* Pure Bash reimplementation. - * - **Build stage 2: Copy XModule framents** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. This is done for a hard-coded list of XModule-style XBlocks, which are not growing in number; it is *not* a problem for in-repository pure XBlock Fragments or pip-installed XBlock assets, which are ready-to-serve. + * - + **Build stage 2: Copy XModule framents** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. This is done for a hard-coded list of XModule-style XBlocks, which are not growing in number; it is *not* a problem for in-repository pure XBlock Fragments or pip-installed XBlock assets, which are ready-to-serve. - ``paver process_xmodule_assets``, or ``xmodule_assets`` @@ -148,7 +148,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* (The initial implementation of build.sh may just point at ``xmodule_assets``). - * - **Build stage 3: Run Webpack** in order to to shim, minify, otherwise process, and bundle JS modules. This requires a call to the npm-installed ``webpack`` binary. + * - + **Build stage 3: Run Webpack** in order to to shim, minify, otherwise process, and bundle JS modules. This requires a call to the npm-installed ``webpack`` binary. - ``paver webpack`` @@ -158,7 +158,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* Bash wrapper around a call to webpack. The script will accept parameters for Django settings rather than looking them up. Open edX distributions, such as Tutor, can choose how to supply the Django-setting-dervied parameters in an efficient manner. - * - **Build stage 4: Compile default SCSS** into CSS for legacy LMS/CMS frontends. + * - + **Build stage 4: Compile default SCSS** into CSS for legacy LMS/CMS frontends. - ``paver compile_sass`` @@ -168,7 +168,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* TODO - * - **Build stage 5: Compiled themes' SCSS** into CSS for legacy LMS/CMS frontends. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. + * - + **Build stage 5: Compiled themes' SCSS** into CSS for legacy LMS/CMS frontends. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. - ``paver compile_sass`` From ca5c0bb4c531419f76277e4c3548caa10f6ebe2a Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 17 Feb 2023 09:55:25 -0500 Subject: [PATCH 23/30] squash: table done? --- docs/decisions/0017-assets-without-python.rst | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index 636bacba0315..17298c78af17 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -128,13 +128,13 @@ The three top-level edx-platform asset processing actions are *build*, *collect* - ``paver update_assets --skip-collect`` - Implemented in Python within update_assets. There is not standalone command for it. + Implemented in Python within update_assets. There is no standalone command for it. - ``assets/build.sh npm`` Pure Bash reimplementation. - * - + **Build stage 2: Copy XModule framents** from the xmodule source tree over to places where will be available for Webpacking and SCSS compliation. This is done for a hard-coded list of XModule-style XBlocks, which are not growing in number; it is *not* a problem for in-repository pure XBlock Fragments or pip-installed XBlock assets, which are ready-to-serve. + * - + **Build stage 2: Copy XModule framents** from the xmodule source tree over to input directories for Webpack and SCSS compilation. This is required for a hard-coded list of old XModule-style XBlocks. This is not required for new pure XBlocks, which include (or pip-install) their assets into edx-platform as ready-to-serve JS/CSS/etc fragments. - ``paver process_xmodule_assets``, or ``xmodule_assets`` @@ -146,7 +146,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* A Bash implementation of XModule asset copying. The aforementioned attributes will be moved from the XModule-style XBlock classes into a simple static JSON file, which the Bash script will be able to read. - (The initial implementation of build.sh may just point at ``xmodule_assets``). + The initial implementation of build.sh may just point at ``xmodule_assets``. * - + **Build stage 3: Run Webpack** in order to to shim, minify, otherwise process, and bundle JS modules. This requires a call to the npm-installed ``webpack`` binary. @@ -160,45 +160,57 @@ The three top-level edx-platform asset processing actions are *build*, *collect* * - + **Build stage 4: Compile default SCSS** into CSS for legacy LMS/CMS frontends. - - ``paver compile_sass`` + - ``paver compile_sass`` - TODO. Mention libsass. + Paver task that invokes ``sass.compile`` (from the libsass Python package) and ``rtlcss`` (installed by npm) for several different directories of SCSS. + + Note: libsass is pinned to a 2015 version with a non-trivial upgrade path. Installing it requires compiling a large C extension, noticably affecting Docker image build time. - ``assets/build.sh common`` - TODO + Bash reimplementation, calling ``node-sass`` and ``rtlcss``. + The initial implementation of build.sh may use ``sassc``, a CLI provided by libsass, instead of node-sass. Then, ``sassc`` can be replaced by ``node-sass`` as part of a subsequent frontend framework upgrade effort. + * - + **Build stage 5: Compiled themes' SCSS** into CSS for legacy LMS/CMS frontends. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. - - ``paver compile_sass`` + - ``./manage.py [lms|cms] compile_sass``, or + ``paver compile_sass --theme-dirs ...`` + + The management command is a wrapper around the paver task. The former looks up the list of theme search directories from Django settings and site configuration; the latter requires them to be supplied as arguments. TODO - - ``assets/build.sh themes`` + - ``./manage.py [lms|cms] compile_sass`` + ``assets/build.sh themes --theme-dirs ...`` - TODO + The management command will remain available, but it will need to be updated to point at the Bash script, which will replace the paver task (see build stage 4 for details). + + The overall asset *build* action will use the Bash script; this means that list of theme directories will need to be provided as arguments, but it ensures that the build can remain Python-free. - * - **Collect** the built static assets from edx-platform to another location (the ``STATIC_ROOT``) so that they can be efficiently served *without* Django's webserver. This step, by nature, requires Python and Django in order to find and organize the assets, which may come from edx-platform itself or from its many installed Python and NPM packages. This is only done for production environments, where it is usually desirable to serve assets with something efficient like NGINX. + * - **Collect** the built static assets from edx-platform to another location (the ``STATIC_ROOT``) so that they can be efficiently served *without* Django's webserver. This step, by nature, requires Python and Django in order to find and organize the assets, which may come from edx-platform itself or from its many installed Python and NPM packages. This is only needed for **production** environments, where it is usually desirable to serve assets with something efficient like NGINX. - ``paver update_assets`` - TODO + Paver task wrapping a call to the standard Django `collectstatic `_ command. It adds ``--noinput`` and a list of ``--ignore`` file patterns to the command call. - - ``./manage.py lms collectstatic && ./manage.py cms collectstatic`` + - ``./manage.py lms collectstatic --noinput && ./manage.py cms collectstatic --noinput`` - TODO + The standard Django interface will be used without a wrapper. The ignore patterns will be added to edx-platform's `staticfiles app configuration `_ so that they do not need to be supplied as part of the command. - * - **Watch** static assets for changes in the background. When a change occurs, rebuild them automatically, so that the Django webserver picks up the changes. This is only necessary in development environments. A few different sets of assets may be watched: XModule assets, Webpack assets, default SCSS, and theme SCSS. + * - **Watch** static assets for changes in the background. When a change occurs, rebuild them automatically, so that the Django webserver picks up the changes. This is only necessary in **development** environments. A few different sets of assets may be watched: XModule fragments, Webpack assets, default SCSS, and theme SCSS. - ``paver watch_assets`` - TODO + Paver task that invokes ``webpack --watch`` for Webpack assets and watchdog (a Python library) for other assets. - ``assets/build.sh --watch `` where ```` if one of the build stages described above - TODO. + Bash wrapprers around invocation(s) of `watchman `_, a popular file-watching library maintained by Meta. Watchman is already installed into edx-platform (and other services) via the pywatchman pip wrapper package. + + Note: This adds a Python dependency to build.sh. However, we could be clear that watchman is an *optional* dependency of build.sh, enabling the optional ``--watch`` feature. This would keep the *build* action Python-free. Alternatively, watchman is also availble Python-free via apt and homebrew. Notes on Tutor ============== From 099a9a5528557098a4991e9f2a702fde413fc76b Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 17 Feb 2023 09:59:16 -0500 Subject: [PATCH 24/30] squash: --- docs/decisions/0017-assets-without-python.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index 17298c78af17..bb97e1a4c045 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -160,7 +160,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* * - + **Build stage 4: Compile default SCSS** into CSS for legacy LMS/CMS frontends. - - ``paver compile_sass`` + - ``paver compile_sass`` Paver task that invokes ``sass.compile`` (from the libsass Python package) and ``rtlcss`` (installed by npm) for several different directories of SCSS. From 2cdb773e67bd4ab1f3fbe0174595eaa5337ef270 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 17 Feb 2023 10:00:48 -0500 Subject: [PATCH 25/30] squash: --- docs/decisions/0017-assets-without-python.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index bb97e1a4c045..868372c68c41 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -175,6 +175,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* * - + **Build stage 5: Compiled themes' SCSS** into CSS for legacy LMS/CMS frontends. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. - ``./manage.py [lms|cms] compile_sass``, or + ``paver compile_sass --theme-dirs ...`` The management command is a wrapper around the paver task. The former looks up the list of theme search directories from Django settings and site configuration; the latter requires them to be supplied as arguments. @@ -182,6 +183,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* TODO - ``./manage.py [lms|cms] compile_sass`` + ``assets/build.sh themes --theme-dirs ...`` The management command will remain available, but it will need to be updated to point at the Bash script, which will replace the paver task (see build stage 4 for details). From 6a1b4e386444d8f0aac7ea066d985af373fa20fd Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 17 Feb 2023 10:04:06 -0500 Subject: [PATCH 26/30] squash: --- docs/decisions/0017-assets-without-python.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index 868372c68c41..edeaf343e5bd 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -137,13 +137,13 @@ The three top-level edx-platform asset processing actions are *build*, *collect* * - + **Build stage 2: Copy XModule framents** from the xmodule source tree over to input directories for Webpack and SCSS compilation. This is required for a hard-coded list of old XModule-style XBlocks. This is not required for new pure XBlocks, which include (or pip-install) their assets into edx-platform as ready-to-serve JS/CSS/etc fragments. - ``paver process_xmodule_assets``, or + ``xmodule_assets`` Equivalent paver task and console script, both pointing at to an application-level Python module. That module inspects attributes from legacy XModule-style XBlock classes in order to determine which static assets to copy and what to name them. - ``assets/build.sh xmodule`` - A Bash implementation of XModule asset copying. The aforementioned attributes will be moved from the XModule-style XBlock classes into a simple static JSON file, which the Bash script will be able to read. The initial implementation of build.sh may just point at ``xmodule_assets``. @@ -180,9 +180,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* The management command is a wrapper around the paver task. The former looks up the list of theme search directories from Django settings and site configuration; the latter requires them to be supplied as arguments. - TODO - - - ``./manage.py [lms|cms] compile_sass`` + - ``./manage.py [lms|cms] compile_sass``, or ``assets/build.sh themes --theme-dirs ...`` @@ -196,6 +194,8 @@ The three top-level edx-platform asset processing actions are *build*, *collect* Paver task wrapping a call to the standard Django `collectstatic `_ command. It adds ``--noinput`` and a list of ``--ignore`` file patterns to the command call. + (This command also builds assets. The *collect* action could not be run on its own without calling pavelib's Python interface.) + - ``./manage.py lms collectstatic --noinput && ./manage.py cms collectstatic --noinput`` The standard Django interface will be used without a wrapper. The ignore patterns will be added to edx-platform's `staticfiles app configuration `_ so that they do not need to be supplied as part of the command. From 61437edcaf318d421fad9ab5fc5dc2abebd4cf36 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 17 Feb 2023 10:43:14 -0500 Subject: [PATCH 27/30] squash: migration guide & spelling --- docs/decisions/0017-assets-without-python.rst | 84 ++++++++++++------- 1 file changed, 54 insertions(+), 30 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index edeaf343e5bd..0ed81029ad38 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -6,7 +6,7 @@ Status Pending -Will be moved to *Accepted* upon completion of re-implementation. +Will be moved to *Accepted* upon completion of reimplementation. Context ******* @@ -68,26 +68,16 @@ Three particular issues have surfaced in Developer Experience Working Group disc - Potential solution(s) * - edx-platform Docker images are too large and/or take too long to build. - - Switch from large, legacy tooling packages (such as libsass-python and paver) to industry standard, precompiled ones (like node-sass or dart-sass). Remove unneccessary & slow calls to Django management commands. + - Switch from large, legacy tooling packages (such as libsass-python and paver) to industry standard, pre-compiled ones (like node-sass or dart-sass). Remove unnecessary & slow calls to Django management commands. * - edx-platform Docker image layers seem to be rebuilt more often than they should. - Remove all Python dependencies from the static asset build process, such that changes to Python code or requirements do not always have to result in a static asset rebuild. * - In Tutor, using a local copy of edx-platform overwrites the Docker image's pre-installed node_modules and pre-built static assets, requiring developers to reinstall & rebuild in order to get a working platform. - - Better parameterize the input and output paths edx-platform asset build, such that it may search for node_modules outside of edx-platform and generate assets outside of edx-platform. + - Better parameterize the input and output paths edx-platform asset build, such that it may `search for node_modules outside of edx-platform and `generate assets outside of edx-platform `. All of these potential solutions would involve refactoring or entirely replacing parts of the current asset processing system. -WIP: Move these links -===================== - -.. _paver: https://github.com/openedx/tutor/tree/open-release/olive.1/pavelib -.. _openedx-assets: https://github.com/overhangio/tutor/blob/v15.0.0/tutor/templates/build/openedx/bin/openedx-assets. - -* `Finish upgrading frontend frameworks `_ -* `Move node_modules outside of edx-platform in Tutor's openedx image `_ -* `Move static assets outside of edx-platform in Tutor's openedx image `_ - Decision ******** @@ -104,6 +94,9 @@ We will largely reimplement edx-platform's asset processing system. We will aim Consequences ************ +Reimplementation Specification +============================== + The three top-level edx-platform asset processing actions are *build*, *collect*, and *watch*. The build action can be further broken down into five stages. Here is how those actions and stages will be reimplemented: @@ -134,7 +127,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* Pure Bash reimplementation. - * - + **Build stage 2: Copy XModule framents** from the xmodule source tree over to input directories for Webpack and SCSS compilation. This is required for a hard-coded list of old XModule-style XBlocks. This is not required for new pure XBlocks, which include (or pip-install) their assets into edx-platform as ready-to-serve JS/CSS/etc fragments. + * - + **Build stage 2: Copy XModule fragments** from the xmodule source tree over to input directories for Webpack and SCSS compilation. This is required for a hard-coded list of old XModule-style XBlocks. This is not required for new pure XBlocks, which include (or pip-install) their assets into edx-platform as ready-to-serve JS/CSS/etc fragments. - ``paver process_xmodule_assets``, or @@ -156,7 +149,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* - ``assets/build.sh webpack`` - Bash wrapper around a call to webpack. The script will accept parameters for Django settings rather than looking them up. Open edX distributions, such as Tutor, can choose how to supply the Django-setting-dervied parameters in an efficient manner. + Bash wrapper around a call to webpack. The script will accept parameters for Django settings rather than looking them up. Open edX distributions, such as Tutor, can choose how to supply the Django-setting-derived parameters in an efficient manner. * - + **Build stage 4: Compile default SCSS** into CSS for legacy LMS/CMS frontends. @@ -164,13 +157,13 @@ The three top-level edx-platform asset processing actions are *build*, *collect* Paver task that invokes ``sass.compile`` (from the libsass Python package) and ``rtlcss`` (installed by npm) for several different directories of SCSS. - Note: libsass is pinned to a 2015 version with a non-trivial upgrade path. Installing it requires compiling a large C extension, noticably affecting Docker image build time. + Note: libsass is pinned to a 2015 version with a non-trivial upgrade path. Installing it requires compiling a large C extension, noticeably affecting Docker image build time. - ``assets/build.sh common`` Bash reimplementation, calling ``node-sass`` and ``rtlcss``. - The initial implementation of build.sh may use ``sassc``, a CLI provided by libsass, instead of node-sass. Then, ``sassc`` can be replaced by ``node-sass`` as part of a subsequent frontend framework upgrade effort. + The initial implementation of build.sh may use ``sassc``, a CLI provided by libsass, instead of node-sass. Then, ``sassc`` can be replaced by ``node-sass`` as part of a subsequent `edx-platform frontend framework upgrade effort `_. * - + **Build stage 5: Compiled themes' SCSS** into CSS for legacy LMS/CMS frontends. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. @@ -208,26 +201,57 @@ The three top-level edx-platform asset processing actions are *build*, *collect* - ``assets/build.sh --watch `` - where ```` if one of the build stages described above + (where ```` is optionally one of the build stages described above. If provided, only that stage's assets will be watched.) + + Bash wrappers around invocation(s) of `watchman `_, a popular file-watching library maintained by Meta. Watchman is already installed into edx-platform (and other services) via the pywatchman pip wrapper package. + + Note: This adds a Python dependency to build.sh. However, we could be clear that watchman is an *optional* dependency of build.sh which enables the optional ``--watch`` feature. This would keep the *build* action Python-free. Alternatively, watchman is also available Python-free via apt and homebrew. + +Migration +========= + +The old asset processing system is `proposed for deprecation (TODO: link to issue) `_. + +The old and new systems will both be available for at least one named release. Operators will encouraged to try the new asset processing system and report any issues they find. Eventually, the old asset processing system will be entirely removed. - Bash wrapprers around invocation(s) of `watchman `_, a popular file-watching library maintained by Meta. Watchman is already installed into edx-platform (and other services) via the pywatchman pip wrapper package. +Tutor migration guide +--------------------- - Note: This adds a Python dependency to build.sh. However, we could be clear that watchman is an *optional* dependency of build.sh, enabling the optional ``--watch`` feature. This would keep the *build* action Python-free. Alternatively, watchman is also availble Python-free via apt and homebrew. +Tutor provides the `openedx-assets `_ Python script on its edx-platform images for building, collection, and watching. The script uses a mix its own implementation and calls out to edx-platform's paver tasks, avoiding the most troublesome parts of the paver tasks. The script and its interface were the inspiration for the new build.sh that this ADAR describes. -Notes on Tutor -============== +As a consequence of this ADR, Tutor will either need to: -TODO +* reimplement the script as a thin wrapper around the new asset processing commands, or +* deprecate and remove the script. -Deprecation of the old asset processing system -============================================== +Either way, the migration path is straightforward: + +.. list-table:: + :header-rows: 1 + + * - Existing Tutor-provided command + - New upstream command + * - ``openedx-assets build`` + - ``assets/build.sh`` + * - ``openedx-assets npm`` + - ``assets/build.sh npm`` + * - ``openedx-assets xmodule`` + - ``assets/build.sh xmodule`` + * - ``openedx-assets common`` + - ``assets/build.sh common`` + * - ``openedx-assets themes`` + - ``assets/build.sh`` + * - ``openedx-assets collect`` + - ``./manage.py [lms|cms] collectstatic --noinput`` + * - ``openedx-assets watch-themes`` + - ``assets/build.sh --watch themes`` -TODO +The options accepted by ``openedx-assets`` will all be valid inputs to ``assets/build.sh``. -Alternatives Considered -*********************** +Rejected Alternatives +********************* -TODO +* **Avoiding work on edx-platform asset tooling; just wait until all frontends have been replatformed into MFEs**. See Context for why this was rejected. -... +* **Rather than replace paver-based asset processing, try to improve it in place.** The effort required to do this seemed comparable to the effort required to perform a full rewrite, and it would not yield any caching benefits of a Python-free asset pipeline. From 829fe15a89857432cb2378aafc688ee39005776b Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 17 Feb 2023 10:45:21 -0500 Subject: [PATCH 28/30] squash: done? --- docs/decisions/0017-assets-without-python.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-assets-without-python.rst index 0ed81029ad38..fa6cc86d9ff5 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-assets-without-python.rst @@ -4,10 +4,15 @@ Building static assets without Python Status ****** -Pending +Provisional Will be moved to *Accepted* upon completion of reimplementation. +Non-exhaustive list of related PRs: + +* https://github.com/openedx/edx-platform/pull/31736 +* ... + Context ******* @@ -210,7 +215,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* Migration ========= -The old asset processing system is `proposed for deprecation (TODO: link to issue) `_. +The old asset processing system will be `proposed for deprecation (TODO: link to issue) `_ upon provisional acceptance of this ADR. The old and new systems will both be available for at least one named release. Operators will encouraged to try the new asset processing system and report any issues they find. Eventually, the old asset processing system will be entirely removed. From c4b58d3f63c10b29cbb6b588f6b39a57aad7f183 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 17 Feb 2023 10:46:10 -0500 Subject: [PATCH 29/30] squash: rename --- ...thout-python.rst => 0017-reimplement-asset-processing.rst} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename docs/decisions/{0017-assets-without-python.rst => 0017-reimplement-asset-processing.rst} (99%) diff --git a/docs/decisions/0017-assets-without-python.rst b/docs/decisions/0017-reimplement-asset-processing.rst similarity index 99% rename from docs/decisions/0017-assets-without-python.rst rename to docs/decisions/0017-reimplement-asset-processing.rst index fa6cc86d9ff5..2e9484eaa2ee 100644 --- a/docs/decisions/0017-assets-without-python.rst +++ b/docs/decisions/0017-reimplement-asset-processing.rst @@ -1,5 +1,5 @@ -Building static assets without Python -##################################### +Reimplement edx-platform static asset processing +################################################ Status ****** From 6276f363fc57d6ead1887f6cf5b4b2496cd00ba4 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 17 Feb 2023 10:53:46 -0500 Subject: [PATCH 30/30] squash: --- docs/decisions/0017-reimplement-asset-processing.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/decisions/0017-reimplement-asset-processing.rst b/docs/decisions/0017-reimplement-asset-processing.rst index 2e9484eaa2ee..eef472a0ff28 100644 --- a/docs/decisions/0017-reimplement-asset-processing.rst +++ b/docs/decisions/0017-reimplement-asset-processing.rst @@ -1,6 +1,13 @@ Reimplement edx-platform static asset processing ################################################ +Overview +******** + +* edx-platform has a complicated process for managing its static frontend assets. It slows down both developers and site operators. +* We will deprecate the current Python+paver asset processing system in favor of a new Bash implementation. +* After one named release, the deprecated paver system will be removed. + Status ******