Update packman and repo_docs versions (#3293)

Co-authored-by: Ashwin Srinath <shwina@users.noreply.github.com>

Author: shwina

docs/repo.bat

@@ -1,10 +1,37 @@
-@echo off
-
-call "%~dp0tools\packman\python.bat" %~dp0tools\repoman\repoman.py %*
-if %errorlevel% neq 0 ( goto Error )
-
-:Success
-exit /b 0
-
-:Error
-exit /b %errorlevel%
+@echo off
+
+:: Set OMNI_REPO_ROOT early so `repo` bootstrapping can target the repository
+:: root when writing out Python dependencies.
+:: Use SETLOCAL and ENDLOCAL to constrain these variables to this batch file.
+:: Use ENABLEDELAYEDEXPANSION to evaluate the value of PM_PACKAGES_ROOT
+:: at execution time.
+SETLOCAL ENABLEDELAYEDEXPANSION
+set OMNI_REPO_ROOT="%~dp0"
+
+:: Set Packman cache directory early if repo-cache.json is configured
+:: so that the Packman Python version is not fetched from the web.
+IF NOT EXIST "%~dp0repo-cache.json" goto :RepoCacheEnd
+
+:: Read PM_PACKAGES_ROOT from repo-cache.json and make sure it is an absolute path (assume relative to the script directory).
+for /f "usebackq tokens=*" %%i in (`powershell -NoProfile -Command "$PM_PACKAGES_ROOT = (Get-Content '%~dp0repo-cache.json' | ConvertFrom-Json).PM_PACKAGES_ROOT; if ([System.IO.Path]::IsPathRooted($PM_PACKAGES_ROOT)) { Write-Output ('absolute;' + $PM_PACKAGES_ROOT) } else { Write-Output ('relative;' + $PM_PACKAGES_ROOT) }"`) do (
+    for /f "tokens=1,2 delims=;" %%A in ("%%i") do (
+        if /i "%%A" == "relative" (
+            set PM_PACKAGES_ROOT=%~dp0%%B
+        ) else (
+            set PM_PACKAGES_ROOT=%%B
+        )
+    )
+)
+
+:RepoCacheEnd
+
+call "%~dp0tools\packman\python.bat" "%~dp0tools\repoman\repoman.py" %*
+if %errorlevel% neq 0 ( goto Error )
+
+:Success
+ENDLOCAL
+exit /b 0
+
+:Error
+ENDLOCAL
+exit /b %errorlevel%

docs/repo.sh

@@ -2,7 +2,35 @@
 
 set -e
 
+# Set OMNI_REPO_ROOT early so `repo` bootstrapping can target the repository
+# root when writing out Python dependencies.
+export OMNI_REPO_ROOT="$( cd "$(dirname "$0")" ; pwd -P )"
+
+# By default custom caching is disabled in repo_man. But if a repo-cache.json
+# caching configuration file is generated via the `repo cache` command, it's
+# presence will trigger the configuration of custom caching.
+if [[ -f "${OMNI_REPO_ROOT}/repo-cache.json" ]]; then
+    PM_PACKAGES_ROOT=$(grep '"PM_PACKAGES_ROOT"' "${OMNI_REPO_ROOT}/repo-cache.json" | sed 's/.*"PM_PACKAGES_ROOT": "\(.*\)".*/\1/')
+
+    # PM_PACKAGES_ROOT is present in the config file. We set this early
+    # so Packman will reference our cached package repository.
+    if [[ -n "${PM_PACKAGES_ROOT}" ]]; then
+        # Use eval to resolve ~ and perform parameter expansion
+        RESOLVED_PACKAGES_ROOT=$(eval echo "$PM_PACKAGES_ROOT")
+
+        if [[ "${RESOLVED_PACKAGES_ROOT}" != /* ]]; then
+            # PM_PACKAGES_ROOT is not an abs path, assumption is then
+            # that it is a relative path to the repository root.
+            PM_PACKAGES_ROOT="${OMNI_REPO_ROOT}/${RESOLVED_PACKAGES_ROOT}"
+        else
+            PM_PACKAGES_ROOT=${RESOLVED_PACKAGES_ROOT}
+        fi
+        export PM_PACKAGES_ROOT
+    fi
+fi
+
 SCRIPT_DIR=$(dirname ${BASH_SOURCE})
 cd "$SCRIPT_DIR"
 
-exec "tools/packman/python.sh" tools/repoman/repoman.py $@
+# Use "exec" to ensure that environment variables don't accidentally affect other processes.
+exec "tools/packman/python.sh" tools/repoman/repoman.py "$@"

docs/repo.toml

@@ -15,6 +15,8 @@ social_media     = [
     [ "discord", "https://discord.com/channels/1019361803752456192/1161051667945508884" ],
 ]
 
+sphinx_version = "4.5.0.2-py3.10-${platform}"
+
 enhanced_search_enabled = true
 api_output_directory = "api"
 use_fast_doxygen_conversion = true

docs/tools/deps/repo-deps.packman.xml

@@ -1,6 +1,6 @@
 <project toolsVersion="5.0">
   <dependency name="repo_man" linkPath="../../_repo/deps/repo_man">
-    <package name="repo_man" version="1.10.1"/>
+    <package name="repo_man" version="1.71.1"/>
   </dependency>
   <dependency name="repo_docs" linkPath="../../_repo/deps/repo_docs">
     <package name="repo_docs" version="0.10.4"/>

docs/tools/packman/bootstrap/configure.bat

@@ -1,4 +1,4 @@
-:: Copyright 2019 NVIDIA CORPORATION
+:: Copyright 2019-2023 NVIDIA CORPORATION
 ::
 :: Licensed under the Apache License, Version 2.0 (the "License");
 :: you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 :: See the License for the specific language governing permissions and
 :: limitations under the License.
 
-set PM_PACKMAN_VERSION=6.57
+set PM_PACKMAN_VERSION=7.24.4
 
 :: Specify where packman command is rooted
 set PM_INSTALL_PATH=%~dp0..
@@ -59,7 +59,7 @@ if defined PM_PYTHON_EXT (
 	goto PACKMAN
 )
 
-set PM_PYTHON_VERSION=3.7.13-nv1-windows-x86_64
+set PM_PYTHON_VERSION=3.10.5-1-windows-x86_64
 set PM_PYTHON_BASE_DIR=%PM_PACKAGES_ROOT%\python
 set PM_PYTHON_DIR=%PM_PYTHON_BASE_DIR%\%PM_PYTHON_VERSION%
 set PM_PYTHON=%PM_PYTHON_DIR%\python.exe
@@ -95,11 +95,16 @@ if exist "%PM_PYTHON%" (
     if exist "%PM_PYTHON_DIR%" ( rd /s /q "%PM_PYTHON_DIR%" > nul )
 )
 
-:: Perform atomic rename
-rename "%TEMP_FOLDER_NAME%" "%PM_PYTHON_VERSION%" 1> nul
-:: Failure during move, need to clean up and abort
+:: Perform atomic move (allowing overwrite, /y)
+move /y "%TEMP_FOLDER_NAME%" "%PM_PYTHON_DIR%" 1> nul
+:: Verify that python.exe is now where we expect
+if exist "%PM_PYTHON%" goto PACKMAN
+
+:: Wait a second and try again (can help with access denied weirdness)
+timeout /t 1 /nobreak 1> nul
+move /y "%TEMP_FOLDER_NAME%" "%PM_PYTHON_DIR%" 1> nul
 if %errorlevel% neq 0 (
-    echo !!! Error renaming python !!!
+    echo !!! Error moving python %TEMP_FOLDER_NAME% -> %PM_PYTHON_DIR% !!!
     call :CLEAN_UP_TEMP_FOLDER
     goto ERROR
 )
@@ -114,7 +119,7 @@ if defined PM_MODULE_DIR_EXT (
 
 set PM_MODULE=%PM_MODULE_DIR%\run.py
 
-if exist "%PM_MODULE%" goto ENSURE_7ZA
+if exist "%PM_MODULE%" goto END
 
 :: Clean out broken PM_MODULE_DIR if it exists
 if exist "%PM_MODULE_DIR%" ( rd /s /q "%PM_MODULE_DIR%" > nul )
@@ -137,19 +142,6 @@ if %errorlevel% neq 0 (
 
 del "%TARGET%"
 
-:ENSURE_7ZA
-set PM_7Za_VERSION=22.01-1
-set PM_7Za_PATH=%PM_PACKAGES_ROOT%\7za\%PM_7ZA_VERSION%
-if exist "%PM_7Za_PATH%" goto END
-set PM_7Za_PATH=%PM_PACKAGES_ROOT%\chk\7za\%PM_7ZA_VERSION%
-if exist "%PM_7Za_PATH%" goto END
-
-"%PM_PYTHON%" -S -s -u -E "%PM_MODULE%" pull "%PM_MODULE_DIR%\deps.packman.xml"
-if %errorlevel% neq 0 (
-    echo !!! Error fetching packman dependencies !!!
-    goto ERROR
-)
-
 goto END
 
 :ERROR_MKDIR_PACKAGES_ROOT

docs/tools/packman/bootstrap/install_package.py

@@ -19,7 +19,8 @@
 import os
 import stat
 import time
-from typing import Any, Callable
+import hashlib
+from typing import Any, Callable, Union
 
 
 RENAME_RETRY_COUNT = 100
@@ -130,7 +131,24 @@ def rename_folder_with_retry(staging_dir: StagingDirectory, folder_name):
     )
 
 
-def install_package(package_path, install_path):
+def generate_sha256_for_file(file_path: Union[str, os.PathLike]) -> str:
+    """Returns the SHA-256 hex digest for the file at `file_path`"""
+    hash = hashlib.sha256()
+    # Read the file in binary mode and update the hash object with data
+    with open(file_path, "rb") as file:
+        for chunk in iter(lambda: file.read(4096), b""):
+            hash.update(chunk)
+    return hash.hexdigest()
+
+
+def install_common_module(package_path, install_path):
+    COMMON_SHA256 = "ef974608cf903f39dbd3f22b3e26c15a1dd21f6e71d05c1510dcf423128cd7a4"
+    package_sha256 = generate_sha256_for_file(package_path)
+    if package_sha256 != COMMON_SHA256:
+        raise RuntimeError(
+            f"Package at '{package_path}' must have a sha256 of '{COMMON_SHA256}' "
+            f"but was found to have '{package_sha256}'"
+        )
     staging_path, version = os.path.split(install_path)
     with StagingDirectory(staging_path) as staging_dir:
         output_folder = staging_dir.get_temp_folder_path()
@@ -151,4 +169,4 @@ def install_package(package_path, install_path):
     for exec_path in paths_list:
         if os.path.normcase(os.path.normpath(exec_path)) == target_path_np_nc:
             raise RuntimeError(f"packman will not install to executable path '{exec_path}'")
-    install_package(sys.argv[1], target_path_np)
+    install_common_module(sys.argv[1], target_path_np)

docs/tools/packman/packman

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2019-2020 NVIDIA CORPORATION
+# Copyright 2019-2023 NVIDIA CORPORATION
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,11 +18,13 @@ set -eu
 
 if echo ${PM_VERBOSITY-} | grep -i "debug" > /dev/null ; then
 	set -x
+	PM_CURL_SILENT=""
+	PM_WGET_QUIET=""
 else
 	PM_CURL_SILENT="-s -S"
 	PM_WGET_QUIET="--quiet"
 fi
-PM_PACKMAN_VERSION=6.57
+export PM_PACKMAN_VERSION=7.24.4
 
 # This is necessary for newer macOS
 if [ `uname` == 'Darwin' ]; then
@@ -58,17 +60,49 @@ if [ ! -d "$PM_PACKAGES_ROOT" ]; then
 	mkdir -p -m a+rwx "$PM_PACKAGES_ROOT"
 fi
 
+execute_with_retry()
+{
+    # Don't exit on error, we need to handle them
+    set +e
+
+    local CMD="$1"
+    local MAX_TRIES=4
+    local DELAY=2
+    local TRIES=0
+    local exit_code
+
+    while [ $TRIES -lt $MAX_TRIES ]
+    do
+        ((TRIES++))
+        eval $CMD
+        exit_code=$?
+        if [ $exit_code -eq 0 ]; then
+            return 0
+        fi
+
+        if [ $TRIES -lt $MAX_TRIES ]; then
+            echo "Attempt $TRIES failed. Retrying in $DELAY seconds ..."
+            sleep $DELAY
+			DELAY=$((DELAY * DELAY))
+			echo "Retrying ..."
+        fi
+    done
+
+    echo "Command failed after $MAX_TRIES attempts: $CMD"
+    return $exit_code
+}
+
 fetch_file_from_s3()
 {
-	SOURCE=$1
-	SOURCE_URL=http://bootstrap.packman.nvidia.com/$SOURCE
-	TARGET=$2
+	local SOURCE=$1
+	local SOURCE_URL=http://bootstrap.packman.nvidia.com/$SOURCE
+	local TARGET=$2
 	echo "Fetching $SOURCE from bootstrap.packman.nvidia.com ..."
+	local CMD="curl -o $TARGET $SOURCE_URL $PM_CURL_SILENT"
 	if command -v wget >/dev/null 2>&1; then
-		wget $PM_WGET_QUIET -O$TARGET $SOURCE_URL
-	else
-		curl -o $TARGET $SOURCE_URL $PM_CURL_SILENT
+		CMD="wget $PM_WGET_QUIET -O$TARGET $SOURCE_URL"
 	fi
+	execute_with_retry "$CMD"
 }
 
 generate_temp_file_name()
@@ -85,7 +119,7 @@ install_python()
 {
 	PLATFORM=`uname`
 	PROCESSOR=`uname -m`
-	PYTHON_VERSION=3.7.13-nv1
+	PYTHON_VERSION=3.10.5-1
 
 	if [ $PLATFORM == 'Darwin' ]; then
 		PYTHON_PACKAGE=$PYTHON_VERSION-macos-x86_64
@@ -154,20 +188,6 @@ if [ ! -f "$PM_MODULE" ]; then
 	fi
 fi
 
-# Ensure 7za package exists:
-PM_7za_VERSION=22.01-1
-export PM_7za_PATH="$PM_PACKAGES_ROOT/7za/$PM_7za_VERSION"
-if [ ! -d "$PM_7za_PATH" ]; then
-    export PM_7za_PATH="$PM_PACKAGES_ROOT/chk/7za/$PM_7za_VERSION"
-    if [ ! -d "$PM_7za_PATH" ]; then
-        "$PM_PYTHON" -S -s -u -E "$PM_MODULE" pull "$PM_MODULE_DIR/deps.packman.xml"
-        if [ "$?" -ne 0 ]; then
-           echo "Failure while installing required 7za package"
-           exit 1
-        fi
-    fi
-fi
-
 # Generate temporary file name for environment variables:
 PM_VAR_PATH=`mktemp -u -t tmp.$$.pmvars.XXXXXX`
 

docs/tools/packman/packman.cmd

@@ -1,11 +1,12 @@
 :: RUN_PM_MODULE must always be at the same spot for packman update to work (batch reloads file during update!)
-:: [xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx]
+:: [xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx]
 :: Reset errorlevel status (don't inherit from caller)
 @call :ECHO_AND_RESET_ERROR
-:: You can remove the call below if you do your own manual configuration of the dev machines
-call "%~dp0\bootstrap\configure.bat"
 
+:: You can remove this section if you do your own manual configuration of the dev machines
+call :CONFIGURE
 if %errorlevel% neq 0 ( exit /b %errorlevel% )
+
 :: Everything below is mandatory
 if not defined PM_PYTHON goto :PYTHON_ENV_ERROR
 if not defined PM_MODULE goto :MODULE_ENV_ERROR
@@ -46,6 +47,7 @@ exit /b 1
 @echo Error while processing and setting environment variables!
 exit /b 1
 
+:: pad [xxxx]
 :ECHO_AND_RESET_ERROR
 @echo off
 if /I "%PM_VERBOSITY%"=="debug" (
@@ -58,3 +60,30 @@ exit /b 0
 for /f "delims=" %%a in ('%PM_PYTHON% -S -s -u -E -c "import tempfile;file = tempfile.NamedTemporaryFile(mode='w+t', delete=False);print(file.name)"') do (set PM_VAR_PATH=%%a)
 set PM_VAR_PATH_ARG=--var-path="%PM_VAR_PATH%"
 goto :RUN_PM_MODULE
+
+:CONFIGURE
+:: Must capture and set code page to work around issue #279, powershell invocation mutates console font
+:: This issue only happens in Windows CMD shell when using 65001 code page. Some Git Bash implementations
+:: don't support chcp so this workaround is a bit convoluted.
+:: Test for chcp:
+chcp > nul 2>&1
+if %errorlevel% equ 0 (
+	for /f "tokens=2 delims=:" %%a in ('chcp') do (set PM_OLD_CODE_PAGE=%%a)
+) else (
+	call :ECHO_AND_RESET_ERROR
+)
+:: trim leading space (this is safe even when PM_OLD_CODE_PAGE has not been set)
+set PM_OLD_CODE_PAGE=%PM_OLD_CODE_PAGE:~1%
+if "%PM_OLD_CODE_PAGE%" equ "65001" (
+	chcp 437 > nul
+	set PM_RESTORE_CODE_PAGE=1
+)
+call "%~dp0\bootstrap\configure.bat"
+set PM_CONFIG_ERRORLEVEL=%errorlevel%
+if defined PM_RESTORE_CODE_PAGE (
+	:: Restore code page
+	chcp %PM_OLD_CODE_PAGE% > nul
+)
+set PM_OLD_CODE_PAGE=
+set PM_RESTORE_CODE_PAGE=
+exit /b %PM_CONFIG_ERRORLEVEL%