Merge pull request #1 from m0hithreddy/develop

PR for release 1.0.0

CHANGELOG.md

@@ -2,3 +2,13 @@
 
 ### 0.0.1
 - Initial Release
+
+### 1.0.0
+
+Breaking Changes:
+  - The remote object creation through ``RemoteModule`` has seen some changes.
+  - The visibility of few attributes are changed from protected to public.
+
+Add Ons:
+  - ``RpycMem`` and ``RemoteModule`` will accept callables for ``rmem_conn`` parameter
+  - Sessions are introduced through ``RpycMemSession`` class.

README.md

@@ -27,26 +27,29 @@
 3. Share data between processes
 
    *Client 1:*
-
+
+   Using RPyC Memory Session
+
    ```python
    from rpyc_mem.connect import RpycMemConnect
-   from rpyc_mem.client import RemoteModule, RpycMem
+   from rpyc_mem.session import RpycMemSession
    
-   rc = RpycMemConnect('localhost', 18813)
-   ro = RemoteModule(rc)
-   rm = RpycMem(rc, 'unique-key', robj_gen=lambda: ro.list([1, 2]))
+   rses = RpycMemSession('localhost', 18813)
+   rm = rses.rmem('unique-key', robj_gen=lambda: rses.rmod().list([1, 2]))
    
    print(rm)    # [1, 2]
    ```
    *Client 2:*
    
+   Using underlying RPyC Memory classes
+
    ```python
    from rpyc_mem.connect import RpycMemConnect
    from rpyc_mem.client import RemoteModule, RpycMem
    
    rc = RpycMemConnect('localhost', 18813)
-   ro = RemoteModule(rc)
-   rm = RpycMem(rc, 'unique-key', robj_gen=lambda: ro.list([1, 2, 3]))
+   rp = RemoteModule(rc)
+   rm = RpycMem(rc, 'unique-key', robj_gen=lambda: rp().list([1, 2, 3]))
    
    print(rm)    # [1, 2]
    rm.append(3)

docs/api/rpyc_mem.rst

@@ -10,6 +10,7 @@ Subpackages
    rpyc_mem.service
    rpyc_mem.connect
    rpyc_mem.client
+   rpyc_mem.session
    rpyc_mem.utils
 
 Submodules

docs/api/rpyc_mem.session.rst

@@ -0,0 +1,21 @@
+rpyc\_mem.session package
+=========================
+
+Submodules
+----------
+
+rpyc\_mem.session.rpyc\_mem\_session module
+-------------------------------------------
+
+.. automodule:: rpyc_mem.session.rpyc_mem_session
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: rpyc_mem.session
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/user_guide/mem_connect_guide.rst

@@ -27,11 +27,11 @@ and validations. ::
 ``RPyC`` `warns <https://rpyc.readthedocs.io/en/latest/install.html#cross-interpreter-compatibility>`_ against having
 different versions for client and server; when the ``ignore_version`` is ``False``, having different versions will raise
 an exception during ``__init__``. The attributes that are not defined by ``RpycMemConnect`` are searched in the underlying
-RPyC connection object, Ex: ``rc.root`` will invoke ``getattr(self._rmem_conn, 'root')``. The ``RpycMemService``
+RPyC connection object, Ex: ``rc.root`` will invoke ``getattr(self.rpyc_conn, 'root')``. The ``RpycMemService``
 attributes can be accessed as if they were defined under ``RpycMemConnect`` namespace, Ex: ``rc.memoize == rc.root.memoize``
 
 ``RpycMemConnect`` performs some basic error recovery as configured by ``max_retry`` and ``retry_delay``. If you want to
-bypass this you can work with raw connection object ``rc._rmem_conn``. ``RpycMemConnect`` has these additional attributes:
+bypass this you can work with raw connection object ``rc.rpyc_conn``. ``RpycMemConnect`` has these additional attributes:
 
     * ``rc.setup_rmem_conn()`` - Re-setup the connection (attempt-close and open).
     * ``rc.rmem_except_handler()`` - Function decorator for handling the connection errors when working with raw
@@ -48,25 +48,25 @@ The following snippet shows their usage::
 
     # Working with raw connection object
     try:
-        rc._rmem_conn.root.rpyc_version()
+        rc.rpyc_conn.root.rpyc_version()
     except EOFError:
         rc.setup_rmem_conn()    # Re-setup the connection ([attempt] close and open)
-        print(rc._rmem_conn.root.rpyc_version())
+        print(rc.rpyc_conn.root.rpyc_version())
 
     # Recovery from connection failures
-    rc._rmem_conn.close()   # With rc.close() connection errors are no more handled
+    rc.rpyc_conn.close()   # With rc.close() connection errors are no more handled
     print(rc.rpyc_version())
 
     # Using exception handlers when working with raw connection object
-    rc._rmem_conn.close()
+    rc.rpyc_conn.close()
 
     def reconnect_hook():
         print('re-connected')
 
     @rc.rmem_except_handler(on_reconnect=reconnect_hook)
     def rmem_fn():
         """Function that uses rpyc connection object"""
-        print(rc._rmem_conn.root.is_memoized('not_memoized'))
+        print(rc.rpyc_conn.root.is_memoized('not_memoized'))
 
     rmem_fn()
 

docs/user_guide/mem_object_guide.rst

@@ -18,10 +18,10 @@ and ``RpycMem`` instead of creating a new connection. ::
         print('Child Process:')
 
         rc = RpycMemConnect('localhost', 18813)
-        ro = RemoteModule(rc)
+        rp = RemoteModule(rc)
 
         # Generators allow delayed execution (dont create if mapping already exists)
-        rm = RpycMem(rc, unique_key, robj_gen=lambda: ro.list([1, 2, 3]))
+        rm = RpycMem(rc, unique_key, robj_gen=lambda: rp().list([1, 2, 3]))
         print(rm)
         rm.append(3)
 
@@ -32,9 +32,9 @@ and ``RpycMem`` instead of creating a new connection. ::
 
     # Assuming service is running on localhost:18813
     rc = RpycMemConnect('localhost', 18813)
-    ro = RemoteModule(rc)
+    rp = RemoteModule(rc)
 
-    rm = RpycMem(rc, 'unique-key', ro.list([1, 2])) # Either pass robj or robj_gen
+    rm = RpycMem(rc, 'unique-key', rp().list([1, 2])) # Either pass robj or robj_gen
 
     print(rm)
     print(len(rm))  # Python special method lookup
@@ -74,12 +74,12 @@ are shared instead of the entire class ::
 
     # Assuming service is running on localhost:18813
     rc = RpycMemConnect('localhost', 18813)
-    ro = RemoteModule(rc)
+    rp = RemoteModule(rc)
 
 
     class Shared:
-        lock = RpycMem(rc, 'lock-key', robj_gen=lambda: ro.threading.Lock())
-        obj = RpycMem(rc, 'obj-key', robj_gen=lambda: ro.list([1, 2]))
+        lock = RpycMem(rc, 'lock-key', robj_gen=lambda: rp('threading').Lock())
+        obj = RpycMem(rc, 'obj-key', robj_gen=lambda: rp().list([1, 2]))
 
         def __init__(self, title):
             self.title = title
@@ -93,7 +93,7 @@ are shared instead of the entire class ::
             with cls.lock:
                 if isinstance(cls.obj, list):
                     print(cls.obj)
-                    cls.obj.rmem_update(ro.tuple([1, 2]))
+                    cls.obj.rmem_update(rp().tuple([1, 2]))
                 else:
                     print('Oops')
 
@@ -120,19 +120,23 @@ are shared instead of the entire class ::
     Oops
     """
 
+Setting the ``multiprocessing`` start method to ``spawn`` is important on Unix based systems (On Windows and MacOs
+``spawn`` is the default start method) because this causes the file to be reimported, which will create a fresh
+RPyC connection. Otherwise two processes will talk to the server from a similar socket object (socket address), which
+will compromise the data integrity on the server side. However, one can rely on ``RpycMemSession`` and forget about
+the connections being reused in different processes. Refer to ``RPyC Memory Session`` guide for more information.
 
-The proxy objects of ``RpycMem`` class will behave like the original objects in most of the cases. However, they come
-with few limitations. The object proxying idea is inspired from the Tomer Filiba's `Python recipe <https://code.activestate.com/
-recipes/496741-object-proxying/>`_. Consider the below interactive session::
+The object proxying idea is inspired from the Tomer Filiba's `Python recipe <https://code.activestate.com/recipes/
+496741-object-proxying/>`_. The proxy objects of ``RpycMem`` class will behave like the original objects in most of
+the cases. However, they come with few limitations. Consider the below interactive session::
 
     >>> from rpyc_mem.connect import RpycMemConnect
-    >>> from rpyc_mem.client import RemoteModule, RpycMem
+    >>> from rpyc_mem.client import RpycMem
 
     >>> rc = RpycMemConnect('localhost', 18813)
-    >>> ro = RemoteModule(rc)
 
-    >>> rm = RpycMem(rc, 'key1', 1)
-    >>> rm = rm + 1 # rm variable is replaced by int and is garbage collected
+    >>> rm = RpycMem(rc, 'key', 1)
+    >>> rm = rm + 1 # rm variable is replaced by int and the proxy object is garbage collected
     >>> print(rm)
     2
     >>> print(type(rm))

docs/user_guide/mem_service_guide.rst

@@ -31,7 +31,7 @@ kwargs used to run the server can be updated by passing ``server_kwargs`` argume
 
 
 Refer to `RPyC documentation <https://rpyc.readthedocs.io/en/latest/api/utils_server.html>`_ for the servers that come
-inbuilt. If the port is left out, a random port is assigned which can be accessed through ``self._server_obj.port``
+inbuilt. If the port is left out, a random port is assigned which can be accessed through ``self.server_obj.port``
 (this is available only after the service is ran).
 
 In the background, ``RpycMemService`` maintains a mapping between the key and the object in a dict. All management

docs/user_guide/mem_session_guide.rst

@@ -0,0 +1,77 @@
+RPyC Memory Session
+-------------------
+
+Juggling around the ``RpycMemConnect`` object across ``RemoteModule`` and ``RpycMem`` can be redundant. ``RpycMemSession``
+brings all of them under one hood and re-purposes the connection object for different operations. ``RpycMemSession`` also
+solves the problem of two processes using a similar ``RPyC`` connection object; this is documented at detail in this `issue
+<https://github.com/tomerfiliba-org/rpyc/issues/482>`_. In breif, when a process forks a child, child gets the memory snapshot
+of the parent, which includes the underlying socket object of rpyc connection. When both parent and child try to send data to
+the server from same socket address, it corrupts the request message structure as defined by the rpyc protocol. To solve this,
+``RpycMemSession`` passes callable for ``rmem_conn`` parameter of ``RemoteModule`` and ``RpycMem``. This callable acts as
+generator of ``rmem_conn`` object. Every time a connection object is needed the callable is invoked. Now ``RpycMemSession``
+keeps track of the processes and whenever a process change is detected a fresh ``rmem_conn`` object is returned. Consider the
+same example as laid out in ``RPyC Memory`` guide but using ``RpycMemSession``::
+
+    import multiprocessing
+    from multiprocessing import Process
+
+    from rpyc_mem.session import RpycMemSession
+
+    """
+    Assuming service is running on localhost:18813
+
+    Defaults:
+    max_retry=4 # Retry is at session level. In specific at session level for each connection object.
+    retry_delay=3
+    ignore_version=False
+    process_safe=True # Make session to return new connection object upon process change.
+    """
+    rses = RpycMemSession('localhost', 18813)
+
+
+    class Shared:
+        lock = rses.rmem('lock-key', robj_gen=lambda: rses.rmod('threading').Lock())
+        obj = rses.rmem('obj-key', robj_gen=lambda: rses.rmod().list([1, 2]))
+
+        def __init__(self, title):
+            self.title = title
+
+        def describe(self):
+            with self.lock:
+                print('%s: %s' % (self.title, self.obj))
+
+        @classmethod
+        def run(cls):
+            with cls.lock:
+                if isinstance(cls.obj, list):
+                    print(cls.obj)
+                    cls.obj.rmem_update(rses.rmod().tuple([1, 2]))
+                else:
+                    print('Oops')
+
+
+    if __name__ == '__main__':
+        # multiprocessing.set_start_method('spawn')
+
+        proc1 = Process(target=Shared.run)
+        proc2 = Process(target=Shared.run)
+        proc3 = Process(target=Shared('Cool-Class').describe)
+
+        proc1.start()
+        proc2.start()
+        proc3.start()
+
+        proc1.join()
+        proc2.join()
+        proc3.join()
+
+    """
+    Output (varied):
+    Cool-Class: [1, 2]
+    [1, 2]
+    Oops
+    """
+
+
+Note the ``set_start_method`` being commented out (makes difference only on Unix based systems). ``RemoteModule`` of the session
+can be accessed through ``rmod``; It is a singleton object.

docs/user_guide/remote_module_guide.rst

@@ -12,21 +12,21 @@ object for creating remote objects. ::
     # Assuming service running on localhost:18813
     rc = RpycMemConnect('localhost', 18813)
 
-    ro = RemoteModule(rc)
+    rp = RemoteModule(rc)
 
-    rlist = ro.list([1, 2])
+    rlist = rp().list([1, 2])
     for i in rlist:
         print(i)
 
     print(rlist.__class__ == list)
     print(type(rlist) == type([1]))
     print(type(rlist))
 
-    rlock = ro.threading.Lock()
+    rlock = rp('threading').Lock()
     rlock.acquire()
-
     print(rlock.locked())
     rlock.release()
+
     with rlock:
         print('synchronized operation')
 
@@ -42,6 +42,5 @@ object for creating remote objects. ::
     """
 
 
-As part of the module resolution, ``RemoteModule`` first searches in the ``builtins`` of remote (``ro.list == ro.builtins.
-list``), if not resolved then tries to (remote) import the package. Remote generators are simply the callables that return
-remote objects created with ``RemoteModule``.
+``RemoteModule`` supports `importlib.import_module <https://docs.python.org/3/library/importlib.html#importlib.import_module>`_ 
+style imports. When ``module`` parameter is not passed, ``builtins`` is assumed by default.

docs/user_guide/user_guide.rst

@@ -8,3 +8,4 @@ User Guide
    mem_connect_guide
    remote_module_guide
    mem_object_guide
+   mem_session_guide

rpyc_mem/_version.py

@@ -1,3 +1,3 @@
 """Version Information."""
 
-__version__ = '0.0.1'
+__version__ = '1.0.0'

rpyc_mem/client/remote_module.py

@@ -5,28 +5,38 @@ class RemoteModule:
     """
     Expose remote modules to create remote python objects
 
-    :param rpyc_mem.connect.RpycMemConnect rmem_conn: Rpyc memory connection
+    :param Union[rpyc_mem.connect.RpycMemConnect, typing.Callable] rmem_conn: Rpyc memory connection
+     or a callable that returns Rpyc memory connection
 
-    .. automethod:: __getattr__
+    .. automethod:: __call__
     """
 
     def __init__(self, rmem_conn):
         """Initialize RemoteModule with rpyc memory connection"""
         self._rmem_conn = rmem_conn
 
-    def __getattr__(self, name):
+    @property
+    def rmem_conn(self):
         """
-        Return ``builtins``/``modules`` of rpyc memory service hosts. Search in remote
-        ``builtins`` before attempting to import ``name`` module.
+        Return the Rpyc memory connection from ``_rmem_conn`` object. If ``_rmem_conn`` is
+        callable return the result of ``_rmem_conn`` invocation else ``_rmem_conn``.
 
-        :param str name: Name of the remote builtins/module
         :return:
         """
-        # Search in remote builtins
-        try:
-            return getattr(self._rmem_conn.remote_import('builtins'), name)
-        except AttributeError:
-            pass
-
-        # Import 'name' module from rmem host
-        return self._rmem_conn.remote_import(name)
+        if callable(self._rmem_conn):
+            return self._rmem_conn()
+
+        return self._rmem_conn
+
+    def __call__(self, module='builtins', package=None):
+        """
+        Return ``modules`` of rpyc memory service hosts.
+
+        :param str module: The module to import in absolute or relative terms (Ex: pkg.mod, ..mod).
+         Defaults to ``builtins``.
+        :param str package: The package which acts as a base for resolving the module (should be set
+         when relative imports are used)
+
+        :return:
+        """
+        return self.rmem_conn.remote_import(module, package)