Tensorflow: How to replace a node in a calculation graph?

Question

If you have two disjoint graphs, and want to link them, turning this:

x = tf.placeholder(\'float\')
y = f(x)

y = tf.placeholder(\'float\')
z = f(y)
<         


        

Answer 1

TL;DR: If you can define the two computations as Python functions, you should do that. If you can't, there's more advanced functionality in TensorFlow to serialize and import graphs, which allows you to compose graphs from different sources.

One way to do this in TensorFlow is to build the disjoint computations as separate tf.Graph objects, then convert them to serialized protocol buffers using Graph.as_graph_def():

with tf.Graph().as_default() as g_1:
  input = tf.placeholder(tf.float32, name="input")
  y = f(input)
  # NOTE: using identity to get a known name for the output tensor.
  output = tf.identity(y, name="output")

gdef_1 = g_1.as_graph_def()

with tf.Graph().as_default() as g_2:  # NOTE: g_2 not g_1       
  input = tf.placeholder(tf.float32, name="input")
  z = g(input)
  output = tf.identity(y, name="output")

gdef_2 = g_2.as_graph_def()

Then you could compose gdef_1 and gdef_2 into a third graph, using tf.import_graph_def():

with tf.Graph().as_default() as g_combined:
  x = tf.placeholder(tf.float32, name="")

  # Import gdef_1, which performs f(x).
  # "input:0" and "output:0" are the names of tensors in gdef_1.
  y, = tf.import_graph_def(gdef_1, input_map={"input:0": x},
                           return_elements=["output:0"])

  # Import gdef_2, which performs g(y)
  z, = tf.import_graph_def(gdef_2, input_map={"input:0": y},
                           return_elements=["output:0"]

Answer 2

If you want to combine trained models (for example to reuse parts of a pretrained model in a new model), you can use a Saver to save a checkpoint of the first model, then restore that model (entirely or partially) into another model.

For example, say you want to reuse model 1's weights w in model 2, and also convert x from a placeholder to a variable:

with tf.Graph().as_default() as g1:
    x = tf.placeholder('float')
    w = tf.Variable(1., name="w")
    y = x * w
    saver = tf.train.Saver()

with tf.Session(graph=g1) as sess:
    w.initializer.run()
    # train...
    saver.save(sess, "my_model1.ckpt")

with tf.Graph().as_default() as g2:
    x = tf.Variable(2., name="v")
    w = tf.Variable(0., name="w")
    z = x + w
    restorer = tf.train.Saver([w]) # only restore w

with tf.Session(graph=g2) as sess:
    x.initializer.run()  # x now needs to be initialized
    restorer.restore(sess, "my_model1.ckpt") # restores w=1
    print(z.eval())  # prints 3.

Answer 3

Practical example:

import tensorflow as tf
g1 = tf.Graph()
with g1.as_default():
    # set variables/placeholders
    tf.placeholder(tf.int32, [], name='g1_a')
    tf.placeholder(tf.int32, [], name='g1_b')

    # example on exacting tensor by name
    a = g1.get_tensor_by_name('g1_a:0')
    b = g1.get_tensor_by_name('g1_b:0')

    # operation ==>>     c = 2 * 3 = 6
    mul_op = tf.multiply(a, b, name='g1_mul')
    sess = tf.Session()
    g1_mul_results = sess.run(mul_op, feed_dict={'g1_a:0': 2, 'g1_b:0': 3})
    print('graph1 mul = ', g1_mul_results)  # output = 6

    print('\ngraph01 operations/variables:')
    for op in g1.get_operations():
        print(op.name)

g2 = tf.Graph()
with g2.as_default():
    # set variables/placeholders
    tf.import_graph_def(g1.as_graph_def())
    g2_c = tf.placeholder(tf.int32, [], name='g2_c')

    # example on exacting tensor by name
    g1_b = g2.get_tensor_by_name('import/g1_b:0')
    g1_mul = g2.get_tensor_by_name('import/g1_mul:0')

    # operation ==>>
    b = tf.multiply(g1_b, g2_c, name='g2_var_times_g1_a')
    f = tf.multiply(g1_mul, g1_b, name='g1_mul_times_g1_b')

    print('\ngraph01 operations/variables:')
    for op in g2.get_operations():
        print(op.name)
    sess = tf.Session()

    # graph1 variable 'a' times graph2 variable 'c'(graph2)
    ans = sess.run('g2_var_times_g1_a:0', feed_dict={'g2_c:0': 4, 'import/g1_b:0': 5})
    print('\ngraph2 g2_var_times_g1_a = ', ans)  # output = 20

    # graph1 mul_op (a*b) times graph1 variable 'b'
    ans = sess.run('g1_a_times_g1_b:0',
                   feed_dict={'import/g1_a:0': 6, 'import/g1_b:0': 7})
    print('\ngraph2 g1_mul_times_g1_b:0 = ', ans)  # output = (6*7)*7 = 294

''' output
graph1 mul =  6

graph01 operations/variables:
g1_a
g1_b
g1_mul

graph01 operations/variables:
import/g1_a
import/g1_b
import/g1_mul
g2_c
g2_var_times_g1_a
g1_a_times_g1_b

graph2 g2_var_times_g1_a =  20

graph2 g1_a_times_g1_b:0 =  294
'''

reference LINK

Answer 4

It turns out that tf.train.import_meta_graph passes all additional arguments to the underlying import_scoped_meta_graph which has the input_map argument and utilizes it when it gets to it's own (internal) invocation of import_graph_def.

It is not documented, and took me waaaay toooo much time to find it, but it works!