From 7ed5500d9441cd757c4eb3319be1f05f50f6ca98 Mon Sep 17 00:00:00 2001 From: rjzamora Date: Thu, 18 Jan 2024 07:53:44 -0800 Subject: [PATCH] use inspect to deterministically serialize local and lambda funcs --- dask_expr/_util.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/dask_expr/_util.py b/dask_expr/_util.py index e0baa3be9..88ce59578 100644 --- a/dask_expr/_util.py +++ b/dask_expr/_util.py @@ -1,6 +1,7 @@ from __future__ import annotations import functools +import inspect from collections import OrderedDict, UserDict from collections.abc import Hashable, Sequence from types import LambdaType @@ -108,7 +109,17 @@ def _normalize_lambda(func): # ref: https://github.com/cloudpipe/cloudpickle/issues/385 func_str = str(func) if func.__name__ == "" or "" in func_str: - return func_str + try: + # If the function is defined in a file, we can + # use the literal source-code string + return inspect.getsource(func) + except OSError as err: + # Function was defined in the python shell + raise ValueError( + "Dask-expr requires that all lambda and local " + "functions be defined within a source file. " + f"Original error message: {err}" + ) return normalize_object(func)