diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index e2db09d61f409bd..88d0f9e33d83e91 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -151,9 +151,26 @@ def k(x): self.assertEqual( e.exception.msg, 'unindent does not match any outer indentation level') - self.assertEqual(e.exception.offset, 9) + self.assertEqual(e.exception.offset, 2) self.assertEqual(e.exception.text, ' x += 5') + # gh-153837: offset should point to the indentation error, not end of line + indent_error_long_line = b"""\ +def foo(): + x = 1 + x = "some_very_long_expression" +""" + readline = BytesIO(indent_error_long_line).readline + with self.assertRaisesRegex(IndentationError, + "unindent does not match any " + "outer indentation level") as e: + for tok in tokenize.tokenize(readline): + pass + self.assertEqual(e.exception.lineno, 3) + self.assertEqual(e.exception.offset, 3) + self.assertEqual(e.exception.text, + ' x = "some_very_long_expression"') + def test_int(self): # Ordinary integers and binary operators self.check_tokenize("0xff <= 255", """\ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-20-19-55-00.gh-issue-153837.bXYZ12.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-20-19-55-00.gh-issue-153837.bXYZ12.rst new file mode 100644 index 000000000000000..1246b3ac886195e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-20-19-55-00.gh-issue-153837.bXYZ12.rst @@ -0,0 +1 @@ +Fix incorrect column offset in :exc:`IndentationError` raised by the :mod:`tokenize` module. The offset now points to the first non-whitespace character on the line where the indentation error occurs, instead of incorrectly pointing to the end of the line. diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index e6d39e4c7dc8235..7603a0edd49ea39 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -148,7 +148,15 @@ _tokenizer_error(tokenizeriterobject *it) goto exit; } - Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, tok->inp - tok->buf); + Py_ssize_t byte_offset = tok->inp - tok->buf; + if (tok->done == E_DEDENT || tok->done == E_TABSPACE || tok->done == E_TOODEEP) { + const char *line_start = tok->line_start ? tok->line_start : tok->buf; + byte_offset = line_start - tok->buf; + while (byte_offset < size && (tok->buf[byte_offset] == ' ' || tok->buf[byte_offset] == '\t')) { + byte_offset++; + } + } + Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, byte_offset); if (offset == -1) { result = -1; goto exit;