Accessing an attribute of a multiprocessing Proxy of a class

Question

I have a class that I want to share in a read-only fashion with children processes in a pool, so I prepared a proxy of a class but it didn\'t work. The following is a simpli

Answer 1

Here's a less verbose alternative that I found to work well in practice. Not sure if there are any disadvantages.

class TestClass:
    def __init__(self, a):
        self.a = a
    def b(self):
        print self.a

def wrap_test_class(*args, **kwargs):
    obj = TestClass(*args, **kwargs)
    obj.get_a = lambda: obj.a
    return obj

class MyManager(BaseManager): pass

MyManager.register('test', wrap_test_class)

This allows you to access a by calling proxy_object.get_a()

Answer 2

The Proxy objects used by multiprocessing.BaseManager and its sub-classes normally only expose methods from the objects they're referring to, not attributes. Now, there is multiprocessing.Manager().Namespace, which provides a Proxy sub-class that does provide access to attributes, rather than methods. We can create our own Proxy type which inherits from that, which enables access to all our attributes, as well as access to our b function:

from multiprocessing.managers import BaseManager, NamespaceProxy

class TestClass(object):
    def __init__(self, a):
        self.a = a

    def b(self):
        print self.a

class MyManager(BaseManager): pass

class TestProxy(NamespaceProxy):
    # We need to expose the same __dunder__ methods as NamespaceProxy,
    # in addition to the b method.
    _exposed_ = ('__getattribute__', '__setattr__', '__delattr__', 'b')

    def b(self):
        callmethod = object.__getattribute__(self, '_callmethod')
        return callmethod('b')

MyManager.register('test', TestClass, TestProxy)

if __name__ == '__main__':
    manager = MyManager()
    manager.start()
    t = TestClass(1)
    print t.a
    mt = manager.test(2)
    print mt.a
    mt.a = 5
    mt.b()

Output:

1
2
5

Edit:

If you want to be able to dynamically add methods from your original class to a Proxy class, you can do something like this:

from multiprocessing.managers import BaseManager, NamespaceProxy
import inspect

class TestClass(object):
    def __init__(self, a):
        self.a = a

    def b(self):
        print self.a

class AnotherClass(object):
    def __init__(self, a):
        self.a = a

    def c(self):
        print self.a

class MyManager(BaseManager): pass

class ProxyBase(NamespaceProxy):
    _exposed_ = ('__getattribute__', '__setattr__', '__delattr__')

class TestProxy(ProxyBase): pass
class AnotherProxy(ProxyBase): pass


def register_proxy(name, cls, proxy):
    for attr in dir(cls):
        if inspect.ismethod(getattr(cls, attr)) and not attr.startswith("__"):
            proxy._exposed_ += (attr,)
            setattr(proxy, attr, 
                    lambda s: object.__getattribute__(s, '_callmethod')(attr))
    MyManager.register(name, cls, proxy)

register_proxy('test', TestClass, TestProxy)
register_proxy('another', AnotherClass, AnotherProxy)

if __name__ == '__main__':
    manager = MyManager()
    manager.start()
    mt = manager.test(2)
    ma = manager.another(3)
    mt.b()
    ma.c()
    mt.a = 5
    ma.a = 6
    mt.b()
    ma.c()

Answer 3

After spending few hours to reading the source codes, here is the simplest ways to implement the proxy class to expose all attributes and methods:

class TestProxy(NamespaceProxy):
    _exposed_ = tuple(dir(Test))

    def __getattr__(self, name):
        result = super().__getattr__(name)
        if isinstance(result, types.MethodType):
            def wrapper(*args, **kwargs):
                self._callmethod(name, args)
            return wrapper
        return result

BaseManager.register('Test', Test, TestProxy)

manager = BaseManager()
test = manager.Test()

Also, here is an auto proxy method:

def Proxy(target):
    dic = {'types': types}
    exec('''def __getattr__(self, key):
        result = self._callmethod('__getattribute__', (key,))
        if isinstance(result, types.MethodType):
            def wrapper(*args, **kwargs):
                self._callmethod(key, args)
            return wrapper
        return result''', dic)
    proxyName = target.__name__ + "Proxy"
    ProxyType = type(proxyName, (NamespaceProxy,), dic)
    ProxyType._exposed_ = tuple(dir(target))
    return ProxyType

TestProxy = Proxy(Test)
BaseManager.register('Test', Test, TestProxy)

manager = BaseManager()
test = manager.Test()

Answer 4

This is an example of passing parameters (example: __getitem__) or not (example: __len__):

class TestProxy(NamespaceProxy):
    _exposed_ = ('__getattribute__', '__setattr__', '__delattr__','__len__','__getitem__')

    def __len__(self):
        callmethod = object.__getattribute__(self, '_callmethod')
        return callmethod('__len__')
    def __getitem__(self,index):
        callmethod = object.__getattribute__(self, '_callmethod')
        return callmethod('__getitem__',(index,))