Speedup mars deserialization by __new__ (mars-project#3283)

(cherry picked from commit b2b7dc6)

mars/core/entity/tileables.py

@@ -288,7 +288,11 @@ def __init__(self: TileableType, *args, **kwargs):
                 self._chunks = sorted(chunks, key=attrgetter("index"))
         except AttributeError:  # pragma: no cover
             pass
+        self._entities = WeakSet()
+        self._executed_sessions = []
 
+    def __on_deserialize__(self):
+        super(TileableData, self).__on_deserialize__()
         self._entities = WeakSet()
         self._executed_sessions = []
 

mars/dataframe/core.py

@@ -1814,7 +1814,7 @@ def median(
 
 class BaseDataFrameChunkData(LazyMetaChunkData):
     __slots__ = ("_dtypes_value",)
-    _no_copy_attrs_ = ChunkData._no_copy_attrs_ | {"_dtypes"}
+    _no_copy_attrs_ = ChunkData._no_copy_attrs_ | {"_dtypes", "_columns_value"}
 
     # required fields
     _shape = TupleField(
@@ -1851,6 +1851,10 @@ def __init__(
         )
         self._dtypes_value = None
 
+    def __on_deserialize__(self):
+        super(BaseDataFrameChunkData, self).__on_deserialize__()
+        self._dtypes_value = None
+
     def __len__(self):
         return self.shape[0]
 
@@ -1992,6 +1996,12 @@ def __init__(
         self._dtypes_value = None
         self._dtypes_dict = None
 
+    def __on_deserialize__(self):
+        super().__on_deserialize__()
+        self._accessors = dict()
+        self._dtypes_value = None
+        self._dtypes_dict = None
+
     def _get_params(self) -> Dict[str, Any]:
         # params return the properties which useful to rebuild a new tileable object
         return {

mars/serialization/serializables/core.py

@@ -17,7 +17,6 @@
 
 import cloudpickle
 
-from ...utils import no_default
 from ..core import Serializer, Placeholder, buffered
 from .field import Field
 from .field_type import (
@@ -144,6 +143,9 @@ def __init__(self, *args, **kwargs):
         for k, v in values.items():
             fields[k].set(self, v)
 
+    def __on_deserialize__(self):
+        pass
+
     def __repr__(self):
         values = ", ".join(
             [
@@ -169,6 +171,10 @@ def copy(self) -> "Serializable":
 _primitive_serial_cache = weakref.WeakKeyDictionary()
 
 
+class _NoFieldValue:
+    pass
+
+
 class SerializableSerializer(Serializer):
     """
     Leverage DictSerializer to perform serde.
@@ -184,7 +190,7 @@ def _get_field_values(cls, obj: Serializable, fields):
                     value = field.on_serialize(value)
             except AttributeError:
                 # Most field values are not None, serialize by list is more efficient than dict.
-                value = no_default
+                value = _NoFieldValue
             values.append(value)
         return values
 
@@ -203,7 +209,7 @@ def serial(self, obj: Serializable, context: Dict):
 
     @staticmethod
     def _set_field_value(obj: Serializable, field: Field, value):
-        if value is no_default:
+        if value is _NoFieldValue:
             return
         if type(value) is Placeholder:
             if field.on_deserialize is not None:
@@ -224,7 +230,7 @@ def deserial(self, serialized: Tuple, context: Dict, subs: List) -> Serializable
         if type(primitives) is not list:
             primitives = cloudpickle.loads(primitives)
 
-        obj = obj_class()
+        obj = obj_class.__new__(obj_class)
 
         if primitives:
             for field, value in zip(obj_class._PRIMITIVE_FIELDS, primitives):
@@ -233,7 +239,7 @@ def deserial(self, serialized: Tuple, context: Dict, subs: List) -> Serializable
         if obj_class._NON_PRIMITIVE_FIELDS:
             for field, value in zip(obj_class._NON_PRIMITIVE_FIELDS, subs[0]):
                 self._set_field_value(obj, field, value)
-
+        obj.__on_deserialize__()
         return obj
 
 

mars/services/subtask/core.py

@@ -130,6 +130,11 @@ def __init__(
         self._pure_depend_keys = None
         self._repr = None
 
+    def __on_deserialize__(self):
+        super(Subtask, self).__on_deserialize__()
+        self._pure_depend_keys = None
+        self._repr = None
+
     @property
     def expect_band(self):
         if self.expect_bands:

mars/tensor/core.py

@@ -58,6 +58,7 @@ class TensorOrder(Enum):
 
 class TensorChunkData(ChunkData):
     __slots__ = ()
+    _no_copy_attrs_ = ChunkData._no_copy_attrs_ | {"dtype"}
     type_name = "Tensor"
 
     # required fields