Update code for better backward and forward compatibility

dill/_dill.py

@@ -747,31 +747,31 @@ def _create_typemap():
 else:
     _typemap = dict((v, k) for k, v in _reverse_typemap.iteritems())
 
-def _unmarshal(string):
-    return marshal.loads(string)
-
 def _load_type(name):
     return _reverse_typemap[name]
 
+from functools import wraps
+
+def unpack_args(func):
+    func_params = inspect.signature(func).parameters
+    last_param = next(reversed(func_params.values()))
+    if last_param.kind == last_param.VAR_POSITIONAL:
+        return func
+    n = len(func_params)
+
+    @wraps(func)
+    def wrapper(*args):
+        if len(args) > n:
+            warnings.warn("This pickle was created with a different version of dill. "
+                          "Extra arguments to the object constructor were discarded.")
+        return func(*args[:n])
+    return wrapper
+
+@unpack_args
 def _create_type(typeobj, *args):
     return typeobj(*args)
 
-def _create_function(fcode, fglobals, fname=None, fdefaults=None,
-                     fclosure=None, fdict=None, fkwdefaults=None):
-    # same as FunctionType, but enable passing __dict__ to new function,
-    # __dict__ is the storehouse for attributes added after function creation
-    func = FunctionType(fcode, fglobals or dict(), fname, fdefaults, fclosure)
-    if fdict is not None:
-        func.__dict__.update(fdict) #XXX: better copy? option to copy?
-    if fkwdefaults is not None:
-        func.__kwdefaults__ = fkwdefaults
-    # 'recurse' only stores referenced modules/objects in fglobals,
-    # thus we need to make sure that we have __builtins__ as well
-    if "__builtins__" not in func.__globals__:
-        func.__globals__["__builtins__"] = globals()["__builtins__"]
-    # assert id(fglobals) == id(func.__globals__)
-    return func
-
+@unpack_args
 def _create_code(*args):
     if PY3 and hasattr(args[-3], 'encode'): #FIXME: from PY2 fails (optcode)
         args = list(args)
@@ -815,13 +815,15 @@ def _create_code(*args):
     elif len(args) == 15: return CodeType(args[0], *args[2:])
     return CodeType(*args)
 
+@unpack_args
 def _create_ftype(ftypeobj, func, args, kwds):
     if kwds is None:
         kwds = {}
     if args is None:
         args = ()
     return ftypeobj(func, *args, **kwds)
 
+@unpack_args
 def _create_lock(locked, *args): #XXX: ignores 'blocking'
     from threading import Lock
     lock = Lock()
@@ -830,6 +832,7 @@ def _create_lock(locked, *args): #XXX: ignores 'blocking'
             raise UnpicklingError("Cannot acquire lock")
     return lock
 
+@unpack_args
 def _create_rlock(count, owner, *args): #XXX: ignores 'blocking'
     lock = RLockType()
     if owner is not None:
@@ -839,6 +842,7 @@ def _create_rlock(count, owner, *args): #XXX: ignores 'blocking'
     return lock
 
 # thanks to matsjoyce for adding all the different file modes
+@unpack_args
 def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdata): # buffering=0
     # only pickles the handle, not the file contents... good? or StringIO(data)?
     # (for file contents see: http://effbot.org/librarybook/copy-reg.htm)
@@ -931,12 +935,14 @@ class PyObject(ctypes.Structure):
         f.seek(position)
     return f
 
+@unpack_args
 def _create_stringi(value, position, closed):
     f = StringIO(value)
     if closed: f.close()
     else: f.seek(position)
     return f
 
+@unpack_args
 def _create_stringo(value, position, closed):
     f = StringIO()
     if closed: f.close()
@@ -991,18 +997,20 @@ def __ror__(self, a):
 _CELL_EMPTY = Sentinel('_CELL_EMPTY')
 
 if PY3:
+    @unpack_args
     def _create_cell(contents=None):
         if contents is not _CELL_EMPTY:
             value = contents
         return (lambda: value).__closure__[0]
 
 else:
+    @unpack_args
     def _create_cell(contents=None):
         if contents is not _CELL_EMPTY:
             value = contents
         return (lambda: value).func_closure[0]
 
-
+@unpack_args
 def _create_weakref(obj, *args):
     from weakref import ref
     if obj is None: # it's dead
@@ -1013,6 +1021,7 @@ def _create_weakref(obj, *args):
         return ref(UserDict(), *args)
     return ref(obj, *args)
 
+@unpack_args
 def _create_weakproxy(obj, callable=False, *args):
     from weakref import proxy
     if obj is None: # it's dead
@@ -1027,6 +1036,7 @@ def _create_weakproxy(obj, callable=False, *args):
 def _eval_repr(repr_str):
     return eval(repr_str)
 
+@unpack_args
 def _create_array(f, args, state, npdict=None):
    #array = numpy.core.multiarray._reconstruct(*args)
     array = f(*args)
@@ -1035,13 +1045,15 @@ def _create_array(f, args, state, npdict=None):
         array.__dict__.update(npdict)
     return array
 
+@unpack_args
 def _create_dtypemeta(scalar_type):
     if NumpyDType is True: __hook__() # a bit hacky I think
     if scalar_type is None:
         return NumpyDType
     return type(NumpyDType(scalar_type))
 
 if OLD37:
+    @unpack_args
     def _create_namedtuple(name, fieldnames, modulename, defaults=None):
         class_ = _import_module(modulename + '.' + name, safe=True)
         if class_ is not None:
@@ -1051,6 +1063,7 @@ def _create_namedtuple(name, fieldnames, modulename, defaults=None):
         t.__module__ = modulename
         return t
 else:
+    @unpack_args
     def _create_namedtuple(name, fieldnames, modulename, defaults=None):
         class_ = _import_module(modulename + '.' + name, safe=True)
         if class_ is not None:
@@ -1163,6 +1176,9 @@ def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO
             else:
                 pickler.write('0')
 
+#def _unmarshal(string):
+#    return marshal.loads(string)
+#
 #@register(CodeType)
 #def save_code(pickler, obj):
 #    log.info("Co: %s" % obj)
@@ -1905,6 +1921,7 @@ def save_function(pickler, obj):
             # recurse to get all globals referred to by obj
             from .detect import globalvars
             globs_copy = globalvars(obj, recurse=True, builtin=True)
+            globs_copy.setdefault('__builtins__', __builtin__)
 
             # Add the name of the module to the globs dictionary to prevent
             # the duplication of the dictionary. Pickle the unpopulated
@@ -1960,7 +1977,7 @@ def save_function(pickler, obj):
             if state_dict:
                 state = state, state_dict
 
-            _save_with_postproc(pickler, (_create_function, (
+            _save_with_postproc(pickler, (FunctionType, (
                   obj.__code__, globs, obj.__name__, obj.__defaults__,
                   closure
             ), state), obj=obj, postproc_list=postproc_list)
@@ -1973,7 +1990,7 @@ def save_function(pickler, obj):
             if obj.__dict__:
                 postproc_list.append((setattr, (obj, '__dict__', obj.__dict__)))
 
-            _save_with_postproc(pickler, (_create_function, (
+            _save_with_postproc(pickler, (FunctionType, (
                 obj.func_code, globs, obj.func_name, obj.func_defaults,
                 closure
             )), obj=obj, postproc_list=postproc_list)

dill/_shims.py

@@ -264,3 +264,22 @@ def _delattr(cell, name):
 
 _setattr = Getattr(_dill, '_setattr', setattr)
 _delattr = Getattr(_dill, '_delattr', delattr)
+
+
+# Kept for backward compatibility, substituted by bare FunctionType.
+@move_to(_dill)
+def _create_function(fcode, fglobals, fname=None, fdefaults=None,
+                     fclosure=None, fdict=None, fkwdefaults=None):
+    # 'recurse' only stores referenced modules/objects in fglobals,
+    # thus we need to make sure that we have __builtins__ as well
+    if fglobals is None:
+        fglobals = {}
+    fglobals.setdefault('__builtins__', globals()['__builtins__'])
+    # same as FunctionType, but enable passing __dict__ to new function,
+    # __dict__ is the storehouse for attributes added after function creation
+    func = _dill.FunctionType(fcode, fglobals, fname, fdefaults, fclosure)
+    if fdict:
+        func.__dict__.update(fdict) #XXX: better copy? option to copy?
+    if fkwdefaults is not None:
+        func.__kwdefaults__ = fkwdefaults
+    return func

tests/test_functions.py

@@ -8,6 +8,7 @@
 import functools
 import dill
 import sys
+from dill._dill import _create_function
 dill.settings['recurse'] = True
 
 
@@ -102,5 +103,26 @@ def test_functions():
 assert dill.loads(dumped_func_e)(1, 2, 3, e2=4) == 12
 assert dill.loads(dumped_func_e)(1, 2, 3, e2=4, e3=5) == 15''')
 
+def test_legacy_constructor():
+    if is_py3():
+        fields = ('__code__', '__globals__', '__name__', '__defaults__',
+                    '__closure__', '__dict__', '__kwdefaults__')
+    else:
+        fields = ('func_code', 'func_globals', 'func_name', 'func_defaults',
+                    'func_closure', '__dict__')
+    members = [getattr(function_d, f) for f in fields]
+
+    # Old pickle.
+    old_func = _create_function(*members)
+    # Recent pickle (dill>=0.3.5).
+    new_func = _create_function(*members[:5])
+    setattr(new_func, fields[5], members[5])
+    if is_py3():
+        setattr(new_func, fields[6], members[6])
+
+    for field in fields:
+        assert getattr(old_func, field) == getattr(new_func, field)
+
 if __name__ == '__main__':
     test_functions()
+    test_legacy_constructor()