use weakrefs to prevent Julia from garbage-collecting arrays that Python needs; conversions of nothing/None; beginnings of callback support

Author: stevengj

README.md

@@ -111,11 +111,7 @@ interface to provide a data pointer and shape information.)
 
 Conversely, when passing arrays *to* Python, Julia `Array` types are
 converted to `PyObject` types *without* making a copy via NumPy,
-e.g. when passed as `pycall` arguments. **Warning:** If Python creates
-a new reference to an `Array` object and returns it from `pycall`, you
-*must* ensure that the original `Array` object still exists (i.e., is not
-garbage collected) as long as any such "hidden" Python references
-exist.
+e.g. when passed as `pycall` arguments.
 
 #### PyDict
 

src/PyCall.jl

@@ -35,28 +35,18 @@ end
 
 type PyObject
     o::PyPtr # the actual PyObject*
-
-    # For copy-free wrapping of arrays, the PyObject may reference a
-    # pointer to Julia array data.  In this case, we must keep a
-    # reference to the Julia object inside the PyObject so that the
-    # former is not garbage collected before the latter.
-    keep::Any
-
-    function PyObject(o::PyPtr, keep::Any)
-        po = new(o, keep)
+    function PyObject(o::PyPtr)
+        po = new(o)
         finalizer(po, pydecref)
         return po
     end
 end
 
-PyObject(o::PyPtr) = PyObject(o, nothing) # no Julia object to keep
-
 function pydecref(o::PyObject)
     if initialized::Bool # don't decref after pyfinalize!
         ccall(pyfunc(:Py_DecRef), Void, (PyPtr,), o.o)
     end
     o.o = C_NULL
-    o.keep = nothing
     o
 end
 
@@ -99,6 +89,10 @@ MethodWrapperType = PyObject(C_NULL)
 # special function type used in NumPy and SciPy (if available)
 ufuncType = PyObject(C_NULL)
 
+# cache Python None and PyNoneType
+pynothing = PyObject(C_NULL)
+PyNoneType = PyObject(C_NULL)
+
 # Py_SetProgramName needs its argument to persist as long as Python does
 pyprogramname = bytestring("")
 
@@ -116,6 +110,8 @@ function pyinitialize(libpy::Ptr{Void})
     global MethodType
     global MethodWrapperType
     global ufuncType
+    global pynothing
+    global PyNoneType
     if !initialized::Bool
         if finalized::Bool
             # From the Py_Finalize documentation:
@@ -147,6 +143,8 @@ function pyinitialize(libpy::Ptr{Void})
         catch
             ufuncType = PyObject(C_NULL) # NumPy not available
         end
+        pynothing = pybuiltin("None")
+        PyNoneType = types["NoneType"]
     end
     return
 end
@@ -185,14 +183,20 @@ function pyfinalize()
     global TypeType
     global MethodType
     global MethodWrapperType
+    global ufuncType
+    global pynothing
+    global PyNoneType
     if initialized::Bool
-        npyfinalize()
         pydecref(ufuncType)
+        npyfinalize()
+        pydecref(PyNoneType)
+        pydecref(pynothing)
         pydecref(BuiltinFunctionType::PyObject)
         pydecref(TypeType::PyObject)
         pydecref(MethodType::PyObject)
         pydecref(MethodWrapperType::PyObject)
         pydecref(inspect::PyObject)
+        pygc_finalize()
         gc() # collect/decref any remaining PyObjects
         ccall(pyfunc(:Py_Finalize), Void, ())
         dlclose(libpython::Ptr{Void})
@@ -285,6 +289,16 @@ end
 
 #########################################################################
 
+include("gc.jl")
+
+# make a PyObject that embeds a reference to keep, to prevent Julia
+# from garbage-collecting keep until o is finalized.
+PyObject(o::PyPtr, keep::Any) = pyembed(PyObject(o), keep)
+
+#########################################################################
+
+include("callback.jl")
+
 include("conversions.jl")
 
 #########################################################################

src/callback.jl

@@ -0,0 +1,52 @@
+# Passing Julia callback functions to Python routines.
+#
+# Note that this will typically involve two functions: the
+# desired Julia function/closure, and a top-level C-callable
+# wrapper function used with PyCFunction_NewEx -- the latter
+# is called from Python and calls the former as needed.
+
+################################################################
+# mirror of Python API types and constants from methodobject.h
+
+type PyMethodDef
+    ml_name::Ptr{Uint8}
+    ml_meth::Ptr{Void}
+    ml_flags::Cint
+    ml_doc::Ptr{Uint8} # may be NULL
+end
+
+# A PyCFunction is a C function of the form
+#     PyObject *func(PyObject *self, PyObject *args)
+# The first parameter is the "self" function for method, or 
+# for module functions it is the module object.  The second
+# parameter is either a tuple of args (for METH_VARARGS),
+# a single arg (for METH_O), or NULL (for METH_NOARGS).  func
+# must return non-NULL (Py_None is okay) unless there was an
+# error, in which case an exception must have been set.
+
+# ml_flags should be one of:
+const METH_VARARGS = 0x0001 # args are a tuple of arguments
+const METH_NOARGS = 0x0004  # no arguments (NULL argument pointer)
+const METH_O = 0x0008       # single argument (not wrapped in tuple)
+
+################################################################
+
+# Define a Python method/function object from f(PyPtr,PyPtr)::PyPtr.
+# Requires f to be a top-level function.
+function pymethod(f::Function, name::String, flags::Integer)
+    # Python expects the PyMethodDef structure to be a *constant*,
+    # and the strings pointed to therein must also be constants,
+    # so we define anonymous globals to hold these
+    def = gensym("PyMethodDef")
+    defname = gensym("PyMethodDef_ml_name")
+    @eval const $defname = bytestring($name)
+    @eval const $def = PyMethodDef(convert(Ptr{Uint8}, $defname),
+                                   $(cfunction(f, PyPtr, (PyPtr,PyPtr))),
+                                   convert(Cint, $flags),
+                                   convert(Ptr{Uint8}, C_NULL))
+    PyObject(@pycheckn ccall(pyfunc(:PyCFunction_NewEx), PyPtr,
+                             (Ptr{PyMethodDef}, Ptr{Void}, Ptr{Void}),
+                             &eval(def), C_NULL, C_NULL))
+end
+
+################################################################

src/conversions.jl

@@ -25,6 +25,8 @@ PyObject(s::String) = PyObject(@pycheckn ccall(pyfunc(:PyString_FromString),
                                                PyPtr, (Ptr{Uint8},),
                                                bytestring(s)))
 
+PyObject(n::Nothing) = pyincref(pynothing)
+
 # conversions to Julia types from PyObject
 
 convert{T<:Integer}(::Type{T}, po::PyObject) = 
@@ -49,6 +51,8 @@ convert{T<:String}(::Type{T}, po::PyObject) =
   bytestring(@pycheck ccall(pyfunc(:PyString_AsString),
                              Ptr{Uint8}, (PyPtr,), po))
 
+convert(::Type{Nothing}, po::PyObject) = nothing
+
 #########################################################################
 # for automatic conversions, I pass Vector{PyAny}, NTuple{PyAny}, etc.,
 # but since PyAny is an abstract type I need to convert this to Any
@@ -318,6 +322,8 @@ pystring_query(o::PyObject) = pyisinstance(o, :PyString_Type) ? String : None
 
 pyfunction_query(o::PyObject) = pyisinstance(o, :PyFunction_Type) || pyisinstance(o, BuiltinFunctionType) || pyisinstance(o, ufuncType) || pyisinstance(o, TypeType) || pyisinstance(o, MethodType) || pyisinstance(o, MethodWrapperType) ? Function : None
 
+pynone_query(o::PyObject) = pyisinstance(o, PyNoneType) ? Nothing : None
+
 # we check for "items" attr since PyMapping_Check doesn't do this (it only
 # checks for __getitem__) and PyMapping_Check returns true for some 
 # scipy scalar array members, grrr.
@@ -365,6 +371,7 @@ function pytype_query(o::PyObject, default::Type)
     @return_not_None pyfunction_query(o)
     @return_not_None pydict_query(o)
     @return_not_None pysequence_query(o)
+    @return_not_None pynone_query(o)
     return default
 end
 

src/gc.jl

@@ -0,0 +1,49 @@
+# Interactions of the Julia and Python garbage collection:
+#
+#    * When we wrap a Python object in a Julia object jo (e.g. a PyDict),
+#      we keep an explicit PyObject reference inside jo, whose finalizer
+#      decrefs the Python object when it is called.
+#
+#    * When we wrap a Julia object jo inside a Python object po
+#      (e.g a numpy array), we add jo to the pycall_gc dictionary,
+#      keyed by a weak reference to po.  The Python weak reference
+#      allows us to register a callback function that is called 
+#      when po is deallocated, and this callback function removes
+#      jo from pycall_gc so that Julia can garbage-collect it.
+
+pycall_gc = Dict{PyPtr,Any}()
+
+function weakref_callback(callback::PyPtr, wo::PyPtr)
+    global pycall_gc
+    try
+        delete!(pycall_gc::Dict{PyPtr,Any}, wo)
+        # not sure what to do if there is an exception here
+    finally
+        ccall(pyfunc(:Py_DecRef), Void, (PyPtr,), wo)
+    end
+    pyincref(pynothing).o
+end
+
+weakref_callback_obj = PyObject(C_NULL) # weakref_callback Python method
+
+function pygc_finalize()
+    global pycall_gc
+    global weakref_callback_obj
+    pydecref(weakref_callback_obj)
+    pycall_gc::Dict{PyPtr,Any} = Dict{PyPtr,Any}()
+end
+
+# "embed" a reference to jo in po, using the weak-reference mechanism
+function pyembed(po::PyObject, jo::Any)
+    global pycall_gc
+    global weakref_callback_obj
+    if (weakref_callback_obj::PyObject).o == C_NULL
+        weakref_callback_obj::PyObject = pymethod(weakref_callback,
+                                                  "weakref_callback",
+                                                  METH_O)
+    end
+    wo = @pycheckn ccall(pyfunc(:PyWeakref_NewRef), PyPtr, (PyPtr,PyPtr), 
+                         po, weakref_callback_obj)
+    (pycall_gc::Dict{PyPtr,Any})[wo] = jo
+    return po
+end