Added an experimental support of free-threading/no-GIL.

Author: taku910

python/README.md

@@ -209,3 +209,12 @@ with urllib.request.urlopen(
 sp = spm.SentencePieceProcessor(model_proto=model.getvalue())
 print(sp.encode('this is test'))
 ```
+
+### Free Threading support
+Experimental support for no-GIL/Free-Threading has been introduced since v0.2.1. For more details, please refer to [this page](https://py-free-threading.github.io.).
+This operates similarly to how [NumPy](https://numpy.org/devdocs/reference/thread_safety.html#free-threaded-python) handles it.
+
+The C++ library's const and static methods, e.g., encode(), decode() and train(), are designed to work in a non-GIL environment.
+However, non-const methods, e.g., load(), may have potential data race issues, so please ensure you implement appropriate locks beforehand.
+
+While this limitation might be removed in the future, please note that it's not a simple fix, as it would require additional shared locks in C++.

python/pyproject.toml

@@ -28,6 +28,7 @@ classifiers = [
   "Programming Language :: Python :: 3.12",
   "Programming Language :: Python :: 3.13",
   "Programming Language :: Python :: 3.14",
+  "Programming Language :: Python :: Free Threading :: 2 - Beta",
   "Topic :: Text Processing :: Linguistic",
   "Topic :: Software Development :: Libraries :: Python Modules",
 ]

python/setup.py

@@ -21,6 +21,7 @@
 import shutil
 import subprocess
 import sys
+import sysconfig
 from setuptools import Extension, setup
 from setuptools.command.build_ext import build_ext as _build_ext
 
@@ -60,6 +61,10 @@ def is_sentencepiece_installed():
     return False
 
 
+def is_gil_disabled():
+  return sysconfig.get_config_var('Py_GIL_DISABLED')
+
+
 def get_cflags_and_libs(root):
   cflags = ['-std=c++17', '-I' + os.path.join(root, 'include')]
   libs = []
@@ -103,6 +108,8 @@ def build_extension(self, ext):
         libs.append('-Wl,-strip-all')
     if sys.platform == 'linux':
       libs.append('-Wl,-Bsymbolic')
+    if is_gil_disabled():
+      cflags.append('-DPy_GIL_DISABLED')
     print('## cflags={}'.format(' '.join(cflags)))
     print('## libs={}'.format(' '.join(libs)))
     ext.extra_compile_args = cflags
@@ -198,11 +205,17 @@ def get_win_arch():
         '8',
     ])
     cflags = ['/std:c++17', '/I.\\build\\root\\include']
+
     libs = [
         '.\\build\\root\\lib\\sentencepiece.lib',
         '.\\build\\root\\lib\\sentencepiece_train.lib',
     ]
 
+  # on Windows, GIL flag is not set automatically.
+  # https://docs.python.org/3/howto/free-threading-python.html
+  if is_gil_disabled():
+    cflags.append('/DPy_GIL_DISABLED')
+
   SENTENCEPIECE_EXT = Extension(
       'sentencepiece._sentencepiece',
       sources=['src/sentencepiece/sentencepiece_wrap.cxx'],

python/src/sentencepiece/sentencepiece.i

@@ -292,6 +292,12 @@ inline void InitNumThreads(const std::vector<T> &ins, int *num_threads) {
 }  // namespace
 %}
 
+%init %{
+#ifdef Py_GIL_DISABLED
+  PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED);
+#endif
+%}
+
 %exception {
   try {
     $action

python/src/sentencepiece/sentencepiece_wrap.cxx

@@ -10614,6 +10614,11 @@ SWIG_init(void) {
 
   SWIG_InstallConstants(d,swig_const_table);
 
+  
+#ifdef Py_GIL_DISABLED
+  PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED);
+#endif
+  
 #if PY_VERSION_HEX >= 0x03000000
   return m;
 #else