Skip to content

build: break pavelib's process_npm_assets out into shell script #31509

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kdmccormick wants to merge 1 commit into from
Closed

build: break pavelib's process_npm_assets out into shell script #31509

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 1 addition & 83 deletions pavelib/assets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,39 +46,6 @@
path('node_modules'),
]

# A list of NPM installed libraries that should be copied into the common
# static directory.
# If string ends with '/' then all file in the directory will be copied.
NPM_INSTALLED_LIBRARIES = [
'backbone.paginator/lib/backbone.paginator.js',
'backbone/backbone.js',
'bootstrap/dist/js/bootstrap.bundle.js',
'hls.js/dist/hls.js',
'jquery-migrate/dist/jquery-migrate.js',
'jquery.scrollto/jquery.scrollTo.js',
'jquery/dist/jquery.js',
'moment-timezone/builds/moment-timezone-with-data.js',
'moment/min/moment-with-locales.js',
'picturefill/dist/picturefill.js',
'requirejs/require.js',
'underscore.string/dist/underscore.string.js',
'underscore/underscore.js',
'@edx/studio-frontend/dist/',
'which-country/index.js'
]

# A list of NPM installed developer libraries that should be copied into the common
# static directory only in development mode.
NPM_INSTALLED_DEVELOPER_LIBRARIES = [
'sinon/pkg/sinon.js',
'squirejs/src/Squire.js',
]

# Directory to install static vendor files
NPM_JS_VENDOR_DIRECTORY = path('common/static/common/js/vendor')
NPM_CSS_VENDOR_DIRECTORY = path("common/static/common/css/vendor")
NPM_CSS_DIRECTORY = path("common/static/common/css")

# system specific lookup path additions, add sass dirs if one system depends on the sass files for other systems
SASS_LOOKUP_DEPENDENCIES = {
'cms': [path('lms') / 'static' / 'sass' / 'partials', ],
Expand Down Expand Up @@ -604,60 +571,11 @@ def process_npm_assets():
"""
Process vendor libraries installed via NPM.
"""
def copy_vendor_library(library, skip_if_missing=False):
"""
Copies a vendor library to the shared vendor directory.
"""
if library.startswith('node_modules/'):
library_path = library
else:
library_path = f'node_modules/{library}'

if library.endswith('.css') or library.endswith('.css.map'):
vendor_dir = NPM_CSS_VENDOR_DIRECTORY
else:
vendor_dir = NPM_JS_VENDOR_DIRECTORY
if os.path.exists(library_path):
sh('/bin/cp -rf {library_path} {vendor_dir}'.format(
library_path=library_path,
vendor_dir=vendor_dir,
))
elif not skip_if_missing:
raise Exception(f'Missing vendor file {library_path}')

def copy_vendor_library_dir(library_dir, skip_if_missing=False):
"""
Copies all vendor libraries in directory to the shared vendor directory.
"""
library_dir_path = f'node_modules/{library_dir}'
print(f'Copying vendor library dir: {library_dir_path}')
if os.path.exists(library_dir_path):
for dirpath, _, filenames in os.walk(library_dir_path):
for filename in filenames:
copy_vendor_library(os.path.join(dirpath, filename), skip_if_missing=skip_if_missing)

# Skip processing of the libraries if this is just a dry run
if tasks.environment.dry_run:
tasks.environment.info("install npm_assets")
return

# Ensure that the vendor directory exists
NPM_JS_VENDOR_DIRECTORY.mkdir_p()
NPM_CSS_DIRECTORY.mkdir_p()
NPM_CSS_VENDOR_DIRECTORY.mkdir_p()

# Copy each file to the vendor directory, overwriting any existing file.
print("Copying vendor files into static directory")
for library in NPM_INSTALLED_LIBRARIES:
if library.endswith('/'):
copy_vendor_library_dir(library)
else:
copy_vendor_library(library)

# Copy over each developer library too if they have been installed
print("Copying developer vendor files into static directory")
for library in NPM_INSTALLED_DEVELOPER_LIBRARIES:
copy_vendor_library(library, skip_if_missing=True)
sh(cmd('/bin/sh', 'scripts/process-npm-assets.sh'))


@task
Expand Down
123 changes: 123 additions & 0 deletions scripts/process-npm-assets.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
#!/bin/sh
#
# Post-process node_modules.
#
# Run this from the root of edx-platform,
# after node_modules are installed,
# but before trying to compile static assets.
#
# Background:
#
# Some parts of the edx-platform frontend are built to use assets
# from node_modules, as you'd expect. Other (older) parts are not.
# These parts share some dependencies.
#
# We want package.json to be the canonical source of package versioning.
# So, we install modules into node_modules, as is typical.
# Then, we use this script to copy some scripts and styles out of node_modules
# and into the places where the older frontends need them to be.

USAGE="Usage: $0 [ (-n|--node-modules) NODE_MODULES_PATH ]"

# By default, we look for node_modules in the current directory.
# Some Open edX distributions may want node_modules to be located somewhere
# else, so we let this be configured with -n|--node-modules.
NODE_MODULES_PATH="./node_modules"

# Vendor destination paths for assets.
# These are not configurable yet, but that could be changed if necessary.
JS_VENDOR_PATH="./common/static/common/js/vendor"
CSS_VENDOR_PATH="./common/static/common/css/vendor"

# Enable stricter sh behavior.
set -eu

# Parse options.
while [ $# -gt 0 ]; do
case $1 in
-n|--node-modules)
shift
if [ $# -eq 0 ]; then
echo "Error: Missing value for -n/--node-modules"
echo "$USAGE"
exit 1
fi
NODE_MODULES_PATH="$1"
shift
;;
--help)
echo "$USAGE"
exit 0
;;
*)
echo "Error: Unrecognized option: $1"
echo "$USAGE"
exit 1
;;
esac
done

echo "-------------------------------------------------------------------------"
echo "Post-processing npm assets...."
echo " Working directory == '$(pwd)'"
echo " NODE_MODULES_PATH == '$NODE_MODULES_PATH'"
echo " CSS_VENDOR_PATH == '$CSS_VENDOR_PATH'"
echo " JS_VENDOR_PATH == '$JS_VENDOR_PATH'"
echo "-------------------------------------------------------------------------"

# Input validation
if ! [ -d "$NODE_MODULES_PATH" ]; then
echo "Error: not a directory: $NODE_MODULES_PATH"
exit 1
fi
if ! [ -d ./common ]; then
echo "Error: not a directory: ./common"
echo "Hint: $0 must be run from the root of the edx-platform directory!"
exit 1
fi

# Echo lines back to user.
set -x

# Create vendor directories.
mkdir -p common/static/common/js/vendor
mkdir -p common/static/common/css/vendor

# Copy studio-frontend assets into into vendor directory.
find "$NODE_MODULES_PATH/@edx/studio-frontend/dist" \
-type f \! -name \*.css \! -name \*.css.map -print0 | \
xargs --null cp --target-directory="$JS_VENDOR_PATH"
find "$NODE_MODULES_PATH/@edx/studio-frontend/dist" \
-type f \( -name \*.css -o -name \*.css.map \) -print0 | \
xargs --null cp --target-directory="$CSS_VENDOR_PATH"

# Copy certain node_modules libraries into vendor directory.
cp --force \
"$NODE_MODULES_PATH/backbone.paginator/lib/backbone.paginator.js" \
"$NODE_MODULES_PATH/backbone/backbone.js" \
"$NODE_MODULES_PATH/bootstrap/dist/js/bootstrap.bundle.js" \
"$NODE_MODULES_PATH/hls.js/dist/hls.js" \
"$NODE_MODULES_PATH/jquery-migrate/dist/jquery-migrate.js" \
"$NODE_MODULES_PATH/jquery.scrollto/jquery.scrollTo.js" \
"$NODE_MODULES_PATH/jquery/dist/jquery.js" \
"$NODE_MODULES_PATH/moment-timezone/builds/moment-timezone-with-data.js" \
"$NODE_MODULES_PATH/moment/min/moment-with-locales.js" \
"$NODE_MODULES_PATH/picturefill/dist/picturefill.js" \
"$NODE_MODULES_PATH/requirejs/require.js" \
"$NODE_MODULES_PATH/underscore.string/dist/underscore.string.js" \
"$NODE_MODULES_PATH/underscore/underscore.js" \
"$NODE_MODULES_PATH/which-country/index.js" \
"$JS_VENDOR_PATH"

# Copy certain node_modules developer libraries into vendor directory.
# Since they're just developer libraries, they might not exist in a production build pipeline.
# So, let the error pass silently (`... || true`) if the copy fails.
cp --force "$NODE_MODULES_PATH/sinon/pkg/sinon.js" "$JS_VENDOR_PATH" || true
cp --force "$NODE_MODULES_PATH/squirejs/src/Squire.js" "$JS_VENDOR_PATH" || true

# Stop echoing.
set +x

echo "-------------------------------------------------------------------------"
echo "Done post-processing npm assets."
echo "-------------------------------------------------------------------------"