From 3434543288a3057b2f9604d42bc1bdf8055732f8 Mon Sep 17 00:00:00 2001 From: Gregory James Comer Date: Wed, 10 Sep 2025 13:53:33 -0700 Subject: [PATCH 1/2] [Windows] Re-enable tokenizers --- .ci/scripts/setup-windows.ps1 | 7 +++++++ install_requirements.py | 9 ++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.ci/scripts/setup-windows.ps1 b/.ci/scripts/setup-windows.ps1 index 20d29e4f558..533d3bc5af4 100644 --- a/.ci/scripts/setup-windows.ps1 +++ b/.ci/scripts/setup-windows.ps1 @@ -13,6 +13,11 @@ conda activate et # Install test dependencies pip install -r .ci/docker/requirements-ci.txt +# Create a symlink to work around path length issues when building submodules (tokenizers). +Push-Location +New-Item -ItemType SymbolicLink -Path "C:\_et" -Target "$CWD" +cd C:\_et + if ($editable -eq 'true') { install_executorch.bat --editable } else { @@ -22,3 +27,5 @@ if ($LASTEXITCODE -ne 0) { Write-Host "Installation was unsuccessful. Exit code: $LASTEXITCODE." exit $LASTEXITCODE } + +Pop-Location diff --git a/install_requirements.py b/install_requirements.py index 3bd75cad6a3..15b4a23a879 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -112,13 +112,8 @@ def install_requirements(use_pytorch_nightly): LOCAL_REQUIREMENTS = [ "third-party/ao", # We need the latest kernels for fast iteration, so not relying on pypi. - ] + ( - [ - "extension/llm/tokenizers", # TODO(larryliu0820): Setup a pypi package for this. - ] - if sys.platform != "win32" - else [] - ) # TODO(gjcomer): Re-enable when buildable on Windows. + "extension/llm/tokenizers", # TODO(larryliu0820): Setup a pypi package for this. + ] # Install packages directly from local copy instead of pypi. # This is usually not recommended. From ee450c271035888b50514192ec4b4a6725867321 Mon Sep 17 00:00:00 2001 From: Gregory Comer Date: Thu, 11 Sep 2025 14:29:03 -0700 Subject: [PATCH 2/2] Fix powershell symlink script --- .ci/scripts/setup-windows.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/setup-windows.ps1 b/.ci/scripts/setup-windows.ps1 index 533d3bc5af4..dc910bb6b80 100644 --- a/.ci/scripts/setup-windows.ps1 +++ b/.ci/scripts/setup-windows.ps1 @@ -15,7 +15,7 @@ pip install -r .ci/docker/requirements-ci.txt # Create a symlink to work around path length issues when building submodules (tokenizers). Push-Location -New-Item -ItemType SymbolicLink -Path "C:\_et" -Target "$CWD" +New-Item -ItemType SymbolicLink -Path "C:\_et" -Target "$($pwd.Path)" cd C:\_et if ($editable -eq 'true') {