transformers v5 support

.github/workflows/checks.yml

@@ -88,11 +88,9 @@ jobs:
         run: |
           uv lock --check
           uv sync
-      - name: Authenticate HuggingFace CLI
+      - name: Authenticate HuggingFace
         if: env.HF_TOKEN != ''
-        run: |
-          uv pip install huggingface_hub
-          uv run huggingface-cli login --token "$HF_TOKEN"
+        run: uv run python -c "import os; from huggingface_hub import login; login(token=os.environ['HF_TOKEN'])"
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
       - name: Unit Test
@@ -200,11 +198,9 @@ jobs:
         run: |
           uv lock --check
           uv sync
-      - name: Authenticate HuggingFace CLI
+      - name: Authenticate HuggingFace
         if: env.HF_TOKEN != ''
-        run: |
-          uv pip install huggingface_hub
-          uv run huggingface-cli login --token "$HF_TOKEN"
+        run: uv run python -c "import os; from huggingface_hub import login; login(token=os.environ['HF_TOKEN'])"
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
       - name: Test Suite with Coverage Report
@@ -297,11 +293,9 @@ jobs:
         with:
           name: test-coverage
           path: docs/source/_static/coverage
-      - name: Authenticate HuggingFace CLI
+      - name: Authenticate HuggingFace
         if: env.HF_TOKEN != ''
-        run: |
-          uv pip install huggingface_hub
-          uv run huggingface-cli login --token "$HF_TOKEN"
+        run: uv run python -c "import os; from huggingface_hub import login; login(token=os.environ['HF_TOKEN'])"
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
       - name: Build Docs

.gitignore

@@ -22,4 +22,4 @@ docs/source/generated
 # docs/source/_static/model_table
 **.orig
 .venv
-
+.env

.vscode/settings.json

@@ -33,7 +33,7 @@
     "notebook.formatOnSave.enabled": true,
     "pylint.importStrategy": "fromEnvironment",
     "python.testing.pytestArgs": [
-        "transformer_lens",
+        "tests"
     ],
     "python.testing.pytestEnabled": true,
     "rewrap.autoWrap.enabled": true,

demos/ARENA_Content.ipynb

@@ -40,8 +40,8 @@
     "\n",
     "    ipython = get_ipython()\n",
     "    # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n",
-    "    # ipython.magic(\"load_ext autoreload\")\n",
-    "    # ipython.magic(\"autoreload 2\")\n",
+    "    # ipython.run_line_magic(\"load_ext\", \"autoreload\")\n",
+    "    # ipython.run_line_magic(\"autoreload\", \"2\")\n",
     "\n",
     "if IN_GITHUB or IN_COLAB:\n",
     "    %pip install torch\n",
@@ -235,7 +235,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
@@ -244,14 +244,16 @@
        "' I'"
       ]
      },
-     "execution_count": 10,
+     "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "\n",
-    "most_likely_next_tokens = reference_gpt2.tokenizer.batch_decode(logits.argmax(dim=-1)[0])\n",
+    "most_likely_next_tokens = reference_gpt2.tokenizer.batch_decode(\n",
+    "    [[int(t)] for t in logits.argmax(dim=-1)[0].tolist()]\n",
+    ")\n",
     "most_likely_next_tokens[-1]\n",
     "\n"
    ]

demos/Activation_Patching_in_TL_Demo.ipynb

@@ -68,8 +68,8 @@
     "\n",
     "    ipython = get_ipython()\n",
     "    # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n",
-    "    ipython.magic(\"load_ext autoreload\")\n",
-    "    ipython.magic(\"autoreload 2\")"
+    "    ipython.run_line_magic(\"load_ext\", \"autoreload\")\n",
+    "    ipython.run_line_magic(\"autoreload\", \"2\")"
    ]
   },
   {

demos/BERT.ipynb

@@ -29,64 +29,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Running as a Jupyter notebook - intended for development only!\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/var/folders/m3/z6c6rcdj1rbb2jh9vqpgvxg40000gn/T/ipykernel_39188/4022418010.py:26: DeprecationWarning:\n",
-      "\n",
-      "`magic(...)` is deprecated since IPython 0.13 (warning added in 8.1), use run_line_magic(magic_name, parameter_s).\n",
-      "\n",
-      "/var/folders/m3/z6c6rcdj1rbb2jh9vqpgvxg40000gn/T/ipykernel_39188/4022418010.py:27: DeprecationWarning:\n",
-      "\n",
-      "`magic(...)` is deprecated since IPython 0.13 (warning added in 8.1), use run_line_magic(magic_name, parameter_s).\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "# NBVAL_IGNORE_OUTPUT\n",
-    "import os\n",
-    "\n",
-    "# Janky code to do different setup when run in a Colab notebook vs VSCode\n",
-    "DEVELOPMENT_MODE = False\n",
-    "IN_GITHUB = os.getenv(\"GITHUB_ACTIONS\") == \"true\"\n",
-    "try:\n",
-    "    import google.colab\n",
-    "\n",
-    "    IN_COLAB = True\n",
-    "    print(\"Running as a Colab notebook\")\n",
-    "\n",
-    "    # PySvelte is an unmaintained visualization library, use it as a backup if circuitsvis isn't working\n",
-    "    # # Install another version of node that makes PySvelte work way faster\n",
-    "    # !curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash -; sudo apt-get install -y nodejs\n",
-    "    # %pip install git+https://github.com/neelnanda-io/PySvelte.git\n",
-    "except:\n",
-    "    IN_COLAB = False\n",
-    "\n",
-    "if not IN_GITHUB and not IN_COLAB:\n",
-    "    print(\"Running as a Jupyter notebook - intended for development only!\")\n",
-    "    from IPython import get_ipython\n",
-    "\n",
-    "    ipython = get_ipython()\n",
-    "    # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n",
-    "    ipython.magic(\"load_ext autoreload\")\n",
-    "    ipython.magic(\"autoreload 2\")\n",
-    "\n",
-    "if IN_COLAB:\n",
-    "    %pip install transformer_lens\n",
-    "    %pip install circuitsvis"
-   ]
+   "outputs": [],
+   "source": "# NBVAL_IGNORE_OUTPUT\nimport os\n\n# Janky code to do different setup when run in a Colab notebook vs VSCode\nDEVELOPMENT_MODE = False\nIN_GITHUB = os.getenv(\"GITHUB_ACTIONS\") == \"true\"\ntry:\n    import google.colab\n\n    IN_COLAB = True\n    print(\"Running as a Colab notebook\")\n\n    # PySvelte is an unmaintained visualization library, use it as a backup if circuitsvis isn't working\n    # # Install another version of node that makes PySvelte work way faster\n    # !curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash -; sudo apt-get install -y nodejs\n    # %pip install git+https://github.com/neelnanda-io/PySvelte.git\nexcept:\n    IN_COLAB = False\n\nif not IN_GITHUB and not IN_COLAB:\n    print(\"Running as a Jupyter notebook - intended for development only!\")\n    from IPython import get_ipython\n\n    ipython = get_ipython()\n    # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n    ipython.run_line_magic(\"load_ext\", \"autoreload\")\n    ipython.run_line_magic(\"autoreload\", \"2\")\n\nif IN_COLAB:\n    %pip install transformer_lens\n    %pip install circuitsvis"
   },
   {
    "cell_type": "code",
@@ -386,4 +332,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

demos/Colab_Compatibility.ipynb

@@ -41,8 +41,8 @@
     "\n",
     "    ipython = get_ipython()\n",
     "    # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n",
-    "    ipython.magic(\"load_ext autoreload\")\n",
-    "    ipython.magic(\"autoreload 2\")\n",
+    "    ipython.run_line_magic(\"load_ext\", \"autoreload\")\n",
+    "    ipython.run_line_magic(\"autoreload\", \"2\")\n",
     "\n",
     "\n",
     "\n",