clean.sh

#!/usr/bin/env bash

# Set the script mode to "strict".
# http://redsymbol.net/articles/unofficial-bash-strict-mode/ without the fail fast.
set -uo pipefail

# Set up any project based local script variables.
SCRIPT_NAME=$(basename -- "${BASH_SOURCE[0]}")
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
TEMP_FILE=$(mktemp /tmp/"${SCRIPT_NAME}".XXXXXXXXX)

SCRIPT_TITLE="Analyzing project cleanliness"

# Perform any cleanup required by the script.
# shellcheck disable=SC2329
cleanup_function() {

	if [[ ${VERBOSE_MODE} -ne 0 ]]; then
		echo "{Performing clean up for script '${SCRIPT_NAME}'.}"
	fi

	# If the temp file was used, get rid of it.
	if [ -f "${TEMP_FILE}" ]; then
		rm "${TEMP_FILE}"
	fi

	# Restore the current directory.
	popd >/dev/null 2>&1 || exit
}

verbose_echo() {
	echo_text=${1:-}

	if [ "${VERBOSE_MODE}" -ne 0 ]; then
		echo "${echo_text}"
	fi
}

# Start the main part of the script off with a title.
start_process() {
	verbose_echo "${SCRIPT_TITLE}..."
	verbose_echo ""
	verbose_echo "{Saving current directory prior to execution.}"
	if ! pushd "${SCRIPT_DIR}" >"${TEMP_FILE}" 2>&1; then
		cat "${TEMP_FILE}"
		complete_process 1 "Script cannot save the current directory before proceeding."
	fi

	trap cleanup_function EXIT
}

# Simple function to stop the process with information about why it stopped.
complete_process() {
	local SCRIPT_RETURN_CODE=${1}
	local COMPLETE_REASON=${2:-}

	if [ -n "${COMPLETE_REASON}" ]; then
		echo "${COMPLETE_REASON}"
	fi

	if [ "${SCRIPT_RETURN_CODE}" -ne 0 ]; then
		echo ""
		echo "${SCRIPT_TITLE} failed."
	else
		verbose_echo ""
		verbose_echo "${SCRIPT_TITLE} succeeded."
	fi

	exit "${SCRIPT_RETURN_CODE}"
}

# Give the user hints on how the script can be used.
show_usage() {
	local SCRIPT_NAME=$0

	echo "Usage:"
	echo "  $(basename "${SCRIPT_NAME}") [flags]"
	echo ""
	echo "Summary:"
	echo "  Executes the tests for the project."
	echo ""
	echo "Flags:"
	echo "  -f,--force-reset <ver>  Force a reset of the virtual environment, with an optional python version to reset to."
	echo "  -m,--mypy-only          Only run mypy checks and exit."
	echo "  -np,--no-publish        Do not publish project summaries if successful."
	echo "  -ns,--no-sourcery       Do not run any sourcery checks."
	echo "  -nu,--no-upgrades       Do not run checks for upgrades."
	echo "  -nw,--no-workers        Do not use multipel workers when executing tests."
	echo "  -s,--sourcery-only      Only run sourcery checks and exit."
	echo "  --perf                  Collect standard performance metrics."
	echo "  --perf-only             Only collect standard performance metrics."
	echo "  -x,--debug              Display debug information about the script as it executes."
	echo "  -q,--quiet              Do not display detailed information during execution."
	echo "  -h,--help               Display this help text."
	echo ""
	exit 1
}

# Parse the command line.
parse_command_line() {

	PUBLISH_MODE=1
	VERBOSE_MODE=1
	PERFORMANCE_MODE=0
	PERFORMANCE_ONLY_MODE=0
	DEBUG_MODE=0
	MYPY_ONLY_MODE=0
	SOURCERY_ONLY_MODE=0
	NO_SOURCERY_MODE=0
	NO_UPGRADE_MODE=0
	FORCE_RESET_MODE=0
	RESET_PYTHON_VERSION=
	WORKERS_MODE=--workers
	PARAMS=()
	while (("$#")); do
		case "$1" in
		-f | --force-reset)
			FORCE_RESET_MODE=1
			temp_version=${2:-0}
			if [[ ${temp_version} == 3* ]]; then
				RESET_PYTHON_VERSION=${temp_version}
				shift
			fi
			shift
			;;
		-m | --mypy-only)
			MYPY_ONLY_MODE=1
			shift
			;;
		-np | --no-publish)
			PUBLISH_MODE=0
			shift
			;;
		-ns | --no-sourcery)
			NO_SOURCERY_MODE=1
			shift
			;;
		-nu | --no-upgrades)
			NO_UPGRADE_MODE=1
			shift
			;;
		-nw | --no-workers)
			WORKERS_MODE=
			shift
			;;
		-s | --sourcery-only)
			SOURCERY_ONLY_MODE=1
			shift
			;;
		--perf)
			PERFORMANCE_MODE=1
			shift
			;;
		--perf-only)
			PERFORMANCE_ONLY_MODE=1
			PERFORMANCE_MODE=1
			shift
			;;
		-q | --quiet)
			VERBOSE_MODE=0
			shift
			;;
		-x | --debug)
			DEBUG_MODE=1
			shift
			;;
		-h | --help)
			show_usage
			;;
		-*) # unsupported flags
			echo "Error: Unsupported flag ${1}" >&2
			show_usage
			;;
		*) # preserve positional arguments
			PARAMS+=("${1}")
			shift
			;;
		esac
	done

	if [[ ${DEBUG_MODE} -ne 0 ]]; then
		set -x
	fi

	if [[ ${NO_SOURCERY_MODE} -ne 0 ]] && [[ ${SOURCERY_ONLY_MODE} -ne 0 ]]; then
		echo "{Script's no-sourcery mode has precedence over the script's sourcery-only mode. Disabling sourcery only mode.}"
		SOURCERY_ONLY_MODE=0
	fi

	if [[ ${SOURCERY_ONLY_MODE} -ne 0 ]] && [[ ${MYPY_ONLY_MODE} -ne 0 ]] && [[ ${PERFORMANCE_ONLY_MODE} -ne 0 ]]; then
		echo "Options '--perf-only', '-m|--mypy-only' and '-s,--sourcery-only' conflict with each other.  Please choose one and try again."
		exit 1
	fi
}

load_properties_from_file() {

	verbose_echo "{Loading 'project.properties file'...}"
	while IFS='=' read -r key_value; do
		if [[ ${key_value} == \#* ]]; then
			continue
		fi
		key=$(echo "${key_value}" | cut -d '=' -f1)
		value=$(echo "${key_value}" | cut -d '=' -f2-)
		export "${key}=${value}"
	done <"${SCRIPT_DIR}/project.properties"

	if [[ -z ${PYTHON_MODULE_NAME} ]]; then
		complete_process 1 "Property 'PYTHON_MODULE_NAME' must be defined in the project.properties file."
	fi
}

verify_and_install_dependencies() {

	verbose_echo ""
	verbose_echo "{Verifying and installing dependencies.}"
	INSTALL_ARGS=()
	if [[ ${FORCE_RESET_MODE} -ne 0 ]]; then
		INSTALL_ARGS+=("--force-reset")
		if [[ -n ${RESET_PYTHON_VERSION} ]]; then
			INSTALL_ARGS+=("${RESET_PYTHON_VERSION}")
		fi
	fi
	if [[ ${DEBUG_MODE} -ne 0 ]]; then
		# shellcheck disable=SC2068  # Double quote array expansions to avoid re-splitting elements.
		if ! ./install_dependencies.sh -x ${INSTALL_ARGS[@]}; then
			complete_process 1 "{Verification (and installation) of dependencies failed.  Please check your Python installation and try again.}"
		fi
	else
		# shellcheck disable=SC2068  # Double quote array expansions to avoid re-splitting elements.
		if ! ./install_dependencies.sh ${INSTALL_ARGS[@]} >"${TEMP_FILE}" 2>&1; then
			cat "${TEMP_FILE}"
			complete_process 1 "{Verification (and installation) of dependencies failed.  Please check your Python installation and try again.}"
		fi
	fi
}

execute_pre_commit() {

	verbose_echo ""
	verbose_echo "{Executing pre-commit hooks on Python code.}"
	PRE_COMMIT_ARGS=()
	if [[ ${MYPY_ONLY_MODE} -ne 0 ]]; then
		PRE_COMMIT_ARGS+=("mypy")
	fi
	if [[ ${PUBLISH_MODE} -ne 0 ]]; then
		PRE_COMMIT_ARGS+=("--all")
	fi
	echo ""
	if ! pipenv run pre-commit run "${PRE_COMMIT_ARGS[@]}"; then
		complete_process 1 "{Executing pre-commit hooks on Python code failed.}"
	fi
	echo ""
}

load_sourcery_configuration() {

	local sourcery_config_file="${SCRIPT_DIR}"/.sourcery.yaml
	if ! [[ -f ${sourcery_config_file} ]]; then
		verbose_echo "{Sourcery configuration file not present. Skipping Sourcery analysis.}"
		return 1
	fi

	echo ""
	if [[ -z ${SOURCERY_USER_KEY:-} ]]; then
		SOURCERY_BATCH_FILE_PATH=${SCRIPT_DIR}/../sourcery.bat
		if [[ -f ${SOURCERY_BATCH_FILE_PATH} ]]; then
			verbose_echo "{Variable 'SOURCERY_USER_KEY' not defined, but Windows '${SOURCERY_BATCH_FILE_PATH}' script detected.}"
			verbose_echo "{Attempting to load variable 'SOURCERY_USER_KEY' from '${SOURCERY_BATCH_FILE_PATH}' script.}"
			while IFS='=' read -r key_value; do
				key=$(echo "${key_value}" | cut -d '=' -f1)
				value=$(echo "${key_value}" | cut -d '=' -f2-)
				if [[ ${key} == "set SOURCERY_USER_KEY" ]]; then
					export SOURCERY_USER_KEY="${value}"
				fi
			done <"${SCRIPT_DIR}/../sourcery.bat"
			if [[ -z ${SOURCERY_USER_KEY} ]]; then
				complete_process 1 "{Unable to load SOURCERY_USER_KEY value from file.}"
			fi
		else
			complete_process 1 "{Variable 'SOURCERY_USER_KEY' is not defined and no sourceable script detected.}"
		fi
	fi
	return 0
}

execute_sourcery() {

	verbose_echo "{Executing Sourcery static analyzer on Python code.}"
	if ! pipenv run sourcery login --token "${SOURCERY_USER_KEY}"; then
		complete_process 1 "{Login to Sourcery failed.}"
	fi

	if [[ ${PUBLISH_MODE} -ne 0 ]]; then
		verbose_echo "{  Executing Sourcery against full project contents.}"
		if ! pipenv run sourcery review --check "${PYTHON_MODULE_NAME}"; then
			complete_process 1 "{Executing Sourcery on Python code failed.}"
		fi
	else
		verbose_echo "{  Executing Sourcery against changed project contents.}"
		if ! pipenv run sourcery review --check "${PYTHON_MODULE_NAME}" --diff "git diff"; then
			complete_process 1 "{Executing Sourcery on Python code failed.}"
		fi
	fi
}

find_unused_pylint_suppressions() {

	if [[ ${PYTHON_MODULE_NAME} == "pylint_utils" ]]; then
		PYLINT_UTILS_SCRIPT_PATH=(python "${SCRIPT_DIR}/main.py")
	else
		PYLINT_UTILS_SCRIPT_PATH=(pylint_utils)
	fi

	SCAN_FILES=()
	git diff --name-only --staged >"${TEMP_FILE}"
	while IFS= read -r line; do
		if [[ ${line} == *.py ]] && [[ ${line} != "pytest_execute.py" ]] && [[ -f ${line} ]]; then
			SCAN_FILES+=("${line}")
		fi
	done <"${TEMP_FILE}"

	verbose_echo ""
	if [[ -z ${SCAN_FILES[*]} ]]; then
		verbose_echo "{Not executing pylint suppression checker on Python source code. No eligible Python files staged.}"
	else
		verbose_echo "{Executing pylint suppression checker on Python source code.}"
		if ! pipenv run "${PYLINT_UTILS_SCRIPT_PATH[@]}" --ignore-path test/resources -s "${SCAN_FILES[@]}"; then
			complete_process 1 "{Executing reporting of unused pylint suppressions in modified Python source code failed.}"
		fi
	fi
}

publish_analysis_results_if_requested() {

	echo ""
	if [[ ${PUBLISH_MODE} -ne 0 ]]; then
		if [[ ${NO_SOURCERY_MODE} -ne 0 ]]; then
			complete_process 1 "Publish mode requires that all scans are performed, but the sourcery scan was disabled.  Please re-enable sourcery and try again."
		fi

		verbose_echo "{Publishing summaries after successful analysis of project.}"
		if ! ./ptest.sh -p; then
			complete_process 1 "{Publishing summaries failed.}"
		fi
	fi
}

analyze_pylint_suppressions() {

	if [[ ${PYTHON_MODULE_NAME} == "pylint_utils" ]]; then
		PYLINT_UTILS_SCRIPT_PATH=(python "${SCRIPT_DIR}/main.py")
	else
		PYLINT_UTILS_SCRIPT_PATH=(pylint_utils)
	fi

	echo ""
	verbose_echo "{Executing pylint utils analyzer on Python source code to verify suppressions and document them.}"
	if ! pipenv run "${PYLINT_UTILS_SCRIPT_PATH[@]}" --recurse -r "${SCRIPT_DIR}/publish/pylint_suppression.json" "${PYTHON_MODULE_NAME}"; then
		complete_process 1 "{Executing reporting of pylint suppressions in Python source code failed.}"
	fi
}

look_for_upgrades() {

	if [[ ${NO_UPGRADE_MODE} -ne 0 ]]; then
		verbose_echo "{Skipping check for Python package upgrades by request.}"
		return
	fi

	verbose_echo "{Looking for Python package upgrades in Pre-Commit and Pipenv.}"
	if ! ./check_project_dependencies.sh; then
		complete_process 1 "{One or more project dependencies can be updated. Please run './check_project_dependencies.sh --upgrade' to update them.}"
	fi
}

execute_test_suite() {

	echo ""
	verbose_echo "{Executing unit tests on Python code.}"
	if ! ./ptest.sh --coverage ${WORKERS_MODE}; then
		complete_process 1 "{Executing application tests failed.}"
	fi
}

execute_performance_suite() {
	if [[ ! -f "${SCRIPT_DIR}/perf_clean.sh" ]]; then
		complete_process 1 "No performance suite available for this project.  Please check the project documentation for more information."
	fi
	if ! ./perf_clean.sh; then
		complete_process 1 "{Executing performance tests failed.}"
	fi
}

# Parse any command line values.
parse_command_line "$@"

# Clean entrance into the script.
start_process

load_properties_from_file

if ! python.exe -m pip install --upgrade pip >"${TEMP_FILE}" 2>&1; then
	cat "${TEMP_FILE}"
	complete_process 1 "{Cannot ensure pip has been upgraded.  Please check your Python installation and try again.}"
fi

if ! pip install -U pipenv==2025.0.3 >"${TEMP_FILE}" 2>&1; then
	cat "${TEMP_FILE}"
	complete_process 1 "{Cannot ensure pipenv has been upgraded.  Please check your Python installation and try again.}"
fi

verify_and_install_dependencies

if [[ ${SOURCERY_ONLY_MODE} -eq 0 ]] && [[ ${PERFORMANCE_ONLY_MODE} -eq 0 ]]; then
	execute_pre_commit

	if [[ ${MYPY_ONLY_MODE} -ne 0 ]]; then
		complete_process 0
	fi
fi

if [[ ${NO_SOURCERY_MODE} -ne 0 ]] || [[ ${PERFORMANCE_ONLY_MODE} -ne 0 ]]; then
	echo "{Skipping Sourcery static analyzer by request.}"
else
	if load_sourcery_configuration; then
		execute_sourcery
	fi

	if [[ ${SOURCERY_ONLY_MODE} -ne 0 ]]; then
		complete_process 0
	fi
fi

if [[ ${PERFORMANCE_ONLY_MODE} -eq 0 ]]; then
	analyze_pylint_suppressions

	find_unused_pylint_suppressions

	look_for_upgrades

	execute_test_suite

	if ! ./package-release.sh; then
		complete_process 1 "{Packaging of the project failed.}"
	fi

	# if ! pipenv run pyroma -n 10 . >"${TEMP_FILE}" 2>&1; then
	# 	cat "${TEMP_FILE}"
	# 	complete_process 1 "{Executing pyroma on Python code failed.}"
	# fi
fi

if [[ ${PERFORMANCE_MODE} -ne 0 ]]; then
	execute_performance_suite
fi

publish_analysis_results_if_requested

# Normal exit from the script.
complete_process 0