python pickle.dumps AssertionError

Question

I'm trying to pickle a class instance containing two lists of another instances. The instances in the two lists have attributes that refer instances of each other. Here are the classes.

import pickle
from copy import copy

class Graph:
    def __init__(self):
        self.vertices = {}
        self.edges = set()
    def __repr__(self):
        return "\n".join(map(str, sorted(self.vertices, key=lambda v:v.id)))

class Edge:
    def __init__(self, vfrom, vto):
        self.vfrom = vfrom
        self.vto = vto
    def __hash__(self):
        return hash((self.vto, self.vfrom))
    def __repr__(self):
        return str(self.vto.id)
    def __getstate__(self):
        vfrom = copy(self.vfrom)
        vfrom.del_outgoing(self)
        vto = copy(self.vto)
        vto.del_incoming(self)
        self.__dict__.update({"vfrom":vfrom, "vto":vto, })
        return self.__dict__
    def __setstate__(self, state):
        self.__dict__.update(state)
        self.__dict__["vfrom"].add_outgoing(self)
        self.__dict__["vto"].add_incoming(self)

class Vertex:
    def __init__(self, id):
        self.id = id
        self.incoming = set()
        self.outgoing = set()
    def __repr__(self):
        return "Vertex %d -> %s"%(self.id, ", ".join(map(str, self.outgoing)))
    def __hash__(self):
        return hash(self.id)
    def add_incoming(self, edge):
        if not edge in self.incoming:
            self.incoming.add(edge)
    def add_outgoing(self, edge):
        if not edge in self.outgoing:
            self.outgoing.add(edge)
    def del_incoming(self, edge):
        self.incoming.discard(edge)
    def del_outgoing(self, edge):
        self.outgoing.discard(edge)

I got an AssertionError when I pickled a simple graph as follows.

>>> v0 = Vertex(0)
>>> v1 = Vertex(1)
>>> e0to1 = Edge(v0, v1)
>>> v0.add_outgoing(e0to1)
>>> v1.add_incoming(e0to1)
>>> g = Graph()
>>> g.vertices[v0] = v0
>>> g.vertices[v1] = v1
>>> g.edges.add(e0to1)
>>> g.edges.add(e0to1)
>>> v2 = Vertex(2)
>>> e0to2 = Edge(v0, v2)
>>> v0.add_outgoing(e0to2)
>>> v2.add_incoming(e0to2)
>>> g.vertices[v2] = v2
>>> g.edges.add(e0to2)
>>> 
>>> print g
Vertex 0 -> 2, 1
Vertex 1 -> 
Vertex 2 -> 
>>> p = pickle.dumps(g)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/lib/python2.6/pickle.py", line 1366, in dumps
    Pickler(file, protocol).dump(obj)
  File "/usr/lib/python2.6/pickle.py", line 224, in dump
    self.save(obj)
  File "/usr/lib/python2.6/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/usr/lib/python2.6/pickle.py", line 725, in save_inst
    save(stuff)
  File "/usr/lib/python2.6/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/usr/lib/python2.6/pickle.py", line 649, in save_dict
    self._batch_setitems(obj.iteritems())
  File "/usr/lib/python2.6/pickle.py", line 663, in _batch_setitems
    save(v)
  File "/usr/lib/python2.6/pickle.py", line 331, in save
    self.save_reduce(obj=obj, *rv)
  File "/usr/lib/python2.6/pickle.py", line 401, in save_reduce
    save(args)
  File "/usr/lib/python2.6/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/usr/lib/python2.6/pickle.py", line 562, in save_tuple
    save(element)
  File "/usr/lib/python2.6/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/usr/lib/python2.6/pickle.py", line 600, in save_list
    self._batch_appends(iter(obj))
  File "/usr/lib/python2.6/pickle.py", line 615, in _batch_appends
    save(x)
  File "/usr/lib/python2.6/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/usr/lib/python2.6/pickle.py", line 725, in save_inst
    save(stuff)
  File "/usr/lib/python2.6/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/usr/lib/python2.6/pickle.py", line 649, in save_dict
    self._batch_setitems(obj.iteritems())
  File "/usr/lib/python2.6/pickle.py", line 663, in _batch_setitems
    save(v)
  File "/usr/lib/python2.6/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/usr/lib/python2.6/pickle.py", line 725, in save_inst
    save(stuff)
  File "/usr/lib/python2.6/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/usr/lib/python2.6/pickle.py", line 649, in save_dict
    self._batch_setitems(obj.iteritems())
  File "/usr/lib/python2.6/pickle.py", line 663, in _batch_setitems
    save(v)
  File "/usr/lib/python2.6/pickle.py", line 331, in save
    self.save_reduce(obj=obj, *rv)
  File "/usr/lib/python2.6/pickle.py", line 405, in save_reduce
    self.memoize(obj)
  File "/usr/lib/python2.6/pickle.py", line 244, in memoize
    assert id(obj) not in self.memo
AssertionError

I worked when the v2 is removed.

>>> v0 = Vertex(0)
>>> v1 = Vertex(1)
>>> e0to1 = Edge(v0, v1)
>>> v0.outgoing.add(e0to1)
>>> v1.incoming.add(e0to1)
>>> g = Graph()
>>> g.vertices[v0] = v0
>>> g.vertices[v1] = v1
>>> g.edges.add(e0to1)
>>> g.edges.add(e0to1)
>>> import cPickle as pickle
>>> p = pickle.dumps(g)
>>> print pickle.loads(p)
Vertex 0 -> 1
Vertex 1 -> 

Do you have any idea?

Answer 1

There are some kinds of data the older pickle protocol cannot handle. To solve your problem, use pickle.HIGHEST_PROTOCOL

>>> p = pickle.dumps(g)
  File "/usr/lib/python2.6/pickle.py", line 244, in memoize
    assert id(obj) not in self.memo
AssertionError:
>>> p = pickle.dumps(g, pickle.HIGHEST_PROTOCOL)
>>> # No problem!