Check agreement between TF and Vespa execution

6 files changed, 1396 insertions, 1177 deletions

diff --git a/searchlib/src/test/files/integration/tensorflow/mnist_softmax/mnist_sftmax_with_saving.py b/searchlib/src/test/files/integration/tensorflow/mnist_softmax/mnist_sftmax_with_saving.py
new file mode 100644
index 00000000000..a1861a1c981
--- /dev/null
+++ b/searchlib/src/test/files/integration/tensorflow/mnist_softmax/mnist_sftmax_with_saving.py
@@ -0,0 +1,89 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""A very simple MNIST classifier.
+
+See extensive documentation at
+https://www.tensorflow.org/get_started/mnist/beginners
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import sys
+
+from tensorflow.examples.tutorials.mnist import input_data
+
+import tensorflow as tf
+
+FLAGS = None
+
+
+def main(_):
+  # Import data
+  mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True)
+
+  # Create the model
+  x = tf.placeholder(tf.float32, [None, 784])
+  W = tf.Variable(tf.zeros([784, 10]))
+  b = tf.Variable(tf.zeros([10]))
+  y = tf.matmul(x, W) + b
+
+  # Define loss and optimizer
+  y_ = tf.placeholder(tf.float32, [None, 10])
+
+  # The raw formulation of cross-entropy,
+  #
+  #   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),
+  #                                 reduction_indices=[1]))
+  #
+  # can be numerically unstable.
+  #
+  # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw
+  # outputs of 'y', and then average across the batch.
+  cross_entropy = tf.reduce_mean(
+      tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))
+  train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
+
+  sess = tf.InteractiveSession()
+  tf.global_variables_initializer().run()
+  # Train
+  for _ in range(1000):
+    batch_xs, batch_ys = mnist.train.next_batch(100)
+    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})
+
+  # Test trained model
+  correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
+  accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
+  print(sess.run(accuracy, feed_dict={x: mnist.test.images,
+                                      y_: mnist.test.labels}))
+
+  # Save the model
+  export_path = "saved"
+  print('Exporting trained model to ', export_path)
+  builder = tf.saved_model.builder.SavedModelBuilder(export_path)
+  signature = tf.saved_model.signature_def_utils.predict_signature_def(inputs = {'x':x}, outputs = {'y':y})
+  builder.add_meta_graph_and_variables(sess,
+                                       [tf.saved_model.tag_constants.SERVING],
+                                       signature_def_map={'serving_default':signature})
+  builder.save(as_text=True)
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--data_dir', type=str, default='/tmp/tensorflow/mnist/input_data',
+                      help='Directory for storing input data')
+  FLAGS, unparsed = parser.parse_known_args()
+  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
diff --git a/searchlib/src/test/files/integration/tensorflow/model1/saved_model.pbtxt b/searchlib/src/test/files/integration/tensorflow/mnist_softmax/saved/saved_model.pbtxt
index e01688669a1..8100dfd594d 100644
--- a/searchlib/src/test/files/integration/tensorflow/model1/saved_model.pbtxt
+++ b/searchlib/src/test/files/integration/tensorflow/mnist_softmax/saved/saved_model.pbtxt
@@ -237,6 +237,45 @@ meta_graphs {
         }
       }
       op {
+        name: "ConcatV2"
+        input_arg {
+          name: "values"
+          type_attr: "T"
+          number_attr: "N"
+        }
+        input_arg {
+          name: "axis"
+          type_attr: "Tidx"
+        }
+        output_arg {
+          name: "output"
+          type_attr: "T"
+        }
+        attr {
+          name: "N"
+          type: "int"
+          has_minimum: true
+          minimum: 2
+        }
+        attr {
+          name: "T"
+          type: "type"
+        }
+        attr {
+          name: "Tidx"
+          type: "type"
+          default_value {
+            type: DT_INT32
+          }
+          allowed_values {
+            list {
+              type: DT_INT32
+              type: DT_INT64
+            }
+          }
+        }
+      }
+      op {
         name: "Const"
         output_arg {
           name: "output"
@@ -291,14 +330,14 @@ meta_graphs {
         is_commutative: true
       }
       op {
-        name: "Fill"
+        name: "ExpandDims"
         input_arg {
-          name: "dims"
-          type: DT_INT32
+          name: "input"
+          type_attr: "T"
         }
         input_arg {
-          name: "value"
-          type_attr: "T"
+          name: "dim"
+          type_attr: "Tdim"
         }
         output_arg {
           name: "output"
@@ -308,48 +347,28 @@ meta_graphs {
           name: "T"
           type: "type"
         }
-      }
-      op {
-        name: "HashTableV2"
-        output_arg {
-          name: "table_handle"
-          type: DT_RESOURCE
-        }
-        attr {
-          name: "container"
-          type: "string"
-          default_value {
-            s: ""
-          }
-        }
         attr {
-          name: "shared_name"
-          type: "string"
+          name: "Tdim"
+          type: "type"
           default_value {
-            s: ""
+            type: DT_INT32
           }
-        }
-        attr {
-          name: "use_node_name_sharing"
-          type: "bool"
-          default_value {
-            b: false
+          allowed_values {
+            list {
+              type: DT_INT32
+              type: DT_INT64
+            }
           }
         }
-        attr {
-          name: "key_dtype"
-          type: "type"
-        }
-        attr {
-          name: "value_dtype"
-          type: "type"
-        }
-        is_stateful: true
       }
       op {
-        name: "Identity"
+        name: "Fill"
         input_arg {
-          name: "input"
+          name: "dims"
+          type: DT_INT32
+        }
+        input_arg {
+          name: "value"
           type_attr: "T"
         }
         output_arg {
@@ -362,37 +381,17 @@ meta_graphs {
         }
       }
       op {
-        name: "InitializeTableV2"
-        input_arg {
-          name: "table_handle"
-          type: DT_RESOURCE
-        }
-        input_arg {
-          name: "keys"
-          type_attr: "Tkey"
-        }
+        name: "FloorDiv"
         input_arg {
-          name: "values"
-          type_attr: "Tval"
-        }
-        attr {
-          name: "Tkey"
-          type: "type"
-        }
-        attr {
-          name: "Tval"
-          type: "type"
+          name: "x"
+          type_attr: "T"
         }
-        is_stateful: true
-      }
-      op {
-        name: "Log"
         input_arg {
-          name: "x"
+          name: "y"
           type_attr: "T"
         }
         output_arg {
-          name: "y"
+          name: "z"
           type_attr: "T"
         }
         attr {
@@ -403,6 +402,12 @@ meta_graphs {
               type: DT_HALF
               type: DT_FLOAT
               type: DT_DOUBLE
+              type: DT_UINT8
+              type: DT_INT8
+              type: DT_UINT16
+              type: DT_INT16
+              type: DT_INT32
+              type: DT_INT64
               type: DT_COMPLEX64
               type: DT_COMPLEX128
             }
@@ -410,32 +415,19 @@ meta_graphs {
         }
       }
       op {
-        name: "LookupTableFindV2"
-        input_arg {
-          name: "table_handle"
-          type: DT_RESOURCE
-        }
-        input_arg {
-          name: "keys"
-          type_attr: "Tin"
-        }
+        name: "Identity"
         input_arg {
-          name: "default_value"
-          type_attr: "Tout"
+          name: "input"
+          type_attr: "T"
         }
         output_arg {
-          name: "values"
-          type_attr: "Tout"
-        }
-        attr {
-          name: "Tin"
-          type: "type"
+          name: "output"
+          type_attr: "T"
         }
         attr {
-          name: "Tout"
+          name: "T"
           type: "type"
         }
-        is_stateful: true
       }
       op {
         name: "MatMul"
@@ -481,6 +473,35 @@ meta_graphs {
         }
       }
       op {
+        name: "Maximum"
+        input_arg {
+          name: "x"
+          type_attr: "T"
+        }
+        input_arg {
+          name: "y"
+          type_attr: "T"
+        }
+        output_arg {
+          name: "z"
+          type_attr: "T"
+        }
+        attr {
+          name: "T"
+          type: "type"
+          allowed_values {
+            list {
+              type: DT_HALF
+              type: DT_FLOAT
+              type: DT_DOUBLE
+              type: DT_INT32
+              type: DT_INT64
+            }
+          }
+        }
+        is_commutative: true
+      }
+      op {
         name: "Mean"
         input_arg {
           name: "input"
@@ -592,32 +613,6 @@ meta_graphs {
         is_commutative: true
       }
       op {
-        name: "Neg"
-        input_arg {
-          name: "x"
-          type_attr: "T"
-        }
-        output_arg {
-          name: "y"
-          type_attr: "T"
-        }
-        attr {
-          name: "T"
-          type: "type"
-          allowed_values {
-            list {
-              type: DT_HALF
-              type: DT_FLOAT
-              type: DT_DOUBLE
-              type: DT_INT32
-              type: DT_INT64
-              type: DT_COMPLEX64
-              type: DT_COMPLEX128
-            }
-          }
-        }
-      }
-      op {
         name: "NoOp"
       }
       op {
@@ -650,88 +645,6 @@ meta_graphs {
         }
       }
       op {
-        name: "ParseExample"
-        input_arg {
-          name: "serialized"
-          type: DT_STRING
-        }
-        input_arg {
-          name: "names"
-          type: DT_STRING
-        }
-        input_arg {
-          name: "sparse_keys"
-          type: DT_STRING
-          number_attr: "Nsparse"
-        }
-        input_arg {
-          name: "dense_keys"
-          type: DT_STRING
-          number_attr: "Ndense"
-        }
-        input_arg {
-          name: "dense_defaults"
-          type_list_attr: "Tdense"
-        }
-        output_arg {
-          name: "sparse_indices"
-          type: DT_INT64
-          number_attr: "Nsparse"
-        }
-        output_arg {
-          name: "sparse_values"
-          type_list_attr: "sparse_types"
-        }
-        output_arg {
-          name: "sparse_shapes"
-          type: DT_INT64
-          number_attr: "Nsparse"
-        }
-        output_arg {
-          name: "dense_values"
-          type_list_attr: "Tdense"
-        }
-        attr {
-          name: "Nsparse"
-          type: "int"
-          has_minimum: true
-        }
-        attr {
-          name: "Ndense"
-          type: "int"
-          has_minimum: true
-        }
-        attr {
-          name: "sparse_types"
-          type: "list(type)"
-          has_minimum: true
-          allowed_values {
-            list {
-              type: DT_FLOAT
-              type: DT_INT64
-              type: DT_STRING
-            }
-          }
-        }
-        attr {
-          name: "Tdense"
-          type: "list(type)"
-          has_minimum: true
-          allowed_values {
-            list {
-              type: DT_FLOAT
-              type: DT_INT64
-              type: DT_STRING
-            }
-          }
-        }
-        attr {
-          name: "dense_shapes"
-          type: "list(shape)"
-          has_minimum: true
-        }
-      }
-      op {
         name: "Placeholder"
         output_arg {
           name: "output"
@@ -752,22 +665,47 @@ meta_graphs {
         }
       }
       op {
-        name: "Range"
-        input_arg {
-          name: "start"
-          type_attr: "Tidx"
-        }
+        name: "Prod"
         input_arg {
-          name: "limit"
-          type_attr: "Tidx"
+          name: "input"
+          type_attr: "T"
         }
         input_arg {
-          name: "delta"
+          name: "reduction_indices"
           type_attr: "Tidx"
         }
         output_arg {
           name: "output"
-          type_attr: "Tidx"
+          type_attr: "T"
+        }
+        attr {
+          name: "keep_dims"
+          type: "bool"
+          default_value {
+            b: false
+          }
+        }
+        attr {
+          name: "T"
+          type: "type"
+          allowed_values {
+            list {
+              type: DT_FLOAT
+              type: DT_DOUBLE
+              type: DT_INT64
+              type: DT_INT32
+              type: DT_UINT8
+              type: DT_UINT16
+              type: DT_INT16
+              type: DT_INT8
+              type: DT_COMPLEX64
+              type: DT_COMPLEX128
+              type: DT_QINT8
+              type: DT_QUINT8
+              type: DT_QINT32
+              type: DT_HALF
+            }
+          }
         }
         attr {
           name: "Tidx"
@@ -777,8 +715,6 @@ meta_graphs {
           }
           allowed_values {
             list {
-              type: DT_FLOAT
-              type: DT_DOUBLE
               type: DT_INT32
               type: DT_INT64
             }
@@ -786,15 +722,19 @@ meta_graphs {
         }
       }
       op {
-        name: "Reciprocal"
+        name: "RealDiv"
         input_arg {
           name: "x"
           type_attr: "T"
         }
-        output_arg {
+        input_arg {
           name: "y"
           type_attr: "T"
         }
+        output_arg {
+          name: "z"
+          type_attr: "T"
+        }
         attr {
           name: "T"
           type: "type"
@@ -803,6 +743,10 @@ meta_graphs {
               type: DT_HALF
               type: DT_FLOAT
               type: DT_DOUBLE
+              type: DT_UINT8
+              type: DT_INT8
+              type: DT_UINT16
+              type: DT_INT16
               type: DT_INT32
               type: DT_INT64
               type: DT_COMPLEX64
@@ -943,13 +887,54 @@ meta_graphs {
         }
       }
       op {
-        name: "Softmax"
+        name: "Slice"
         input_arg {
-          name: "logits"
+          name: "input"
+          type_attr: "T"
+        }
+        input_arg {
+          name: "begin"
+          type_attr: "Index"
+        }
+        input_arg {
+          name: "size"
+          type_attr: "Index"
+        }
+        output_arg {
+          name: "output"
+          type_attr: "T"
+        }
+        attr {
+          name: "T"
+          type: "type"
+        }
+        attr {
+          name: "Index"
+          type: "type"
+          allowed_values {
+            list {
+              type: DT_INT32
+              type: DT_INT64
+            }
+          }
+        }
+      }
+      op {
+        name: "SoftmaxCrossEntropyWithLogits"
+        input_arg {
+          name: "features"
+          type_attr: "T"
+        }
+        input_arg {
+          name: "labels"
+          type_attr: "T"
+        }
+        output_arg {
+          name: "loss"
           type_attr: "T"
         }
         output_arg {
-          name: "softmax"
+          name: "backprop"
           type_attr: "T"
         }
         attr {
@@ -1011,6 +996,10 @@ meta_graphs {
               type: DT_HALF
               type: DT_FLOAT
               type: DT_DOUBLE
+              type: DT_UINT8
+              type: DT_INT8
+              type: DT_UINT16
+              type: DT_INT16
               type: DT_INT32
               type: DT_INT64
               type: DT_COMPLEX64
@@ -1109,49 +1098,6 @@ meta_graphs {
         }
       }
       op {
-        name: "TopKV2"
-        input_arg {
-          name: "input"
-          type_attr: "T"
-        }
-        input_arg {
-          name: "k"
-          type: DT_INT32
-        }
-        output_arg {
-          name: "values"
-          type_attr: "T"
-        }
-        output_arg {
-          name: "indices"
-          type: DT_INT32
-        }
-        attr {
-          name: "sorted"
-          type: "bool"
-          default_value {
-            b: true
-          }
-        }
-        attr {
-          name: "T"
-          type: "type"
-          allowed_values {
-            list {
-              type: DT_FLOAT
-              type: DT_DOUBLE
-              type: DT_INT32
-              type: DT_INT64
-              type: DT_UINT8
-              type: DT_INT16
-              type: DT_INT8
-              type: DT_UINT16
-              type: DT_HALF
-            }
-          }
-        }
-      }
-      op {
         name: "VariableV2"
         output_arg {
           name: "ref"
@@ -1182,224 +1128,27 @@ meta_graphs {
         }
         is_stateful: true
       }
-    }
-    tags: "serve"
-    tensorflow_version: "1.3.0"
-    tensorflow_git_version: "v1.3.0-rc2-20-g0787eee"
-  }
-  graph_def {
-    node {
-      name: "tf_example"
-      op: "Placeholder"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              unknown_rank: true
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "shape"
-        value {
-          shape {
-            unknown_rank: true
-          }
-        }
-      }
-    }
-    node {
-      name: "ParseExample/Const"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_FLOAT
-            tensor_shape {
-              dim {
-              }
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "ParseExample/ParseExample/names"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-              dim {
-              }
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "ParseExample/ParseExample/dense_keys_0"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-            }
-            string_val: "x"
-          }
-        }
-      }
-    }
-    node {
-      name: "ParseExample/ParseExample"
-      op: "ParseExample"
-      input: "tf_example"
-      input: "ParseExample/ParseExample/names"
-      input: "ParseExample/ParseExample/dense_keys_0"
-      input: "ParseExample/Const"
-      attr {
-        key: "Ndense"
-        value {
-          i: 1
-        }
-      }
-      attr {
-        key: "Nsparse"
-        value {
-          i: 0
-        }
-      }
-      attr {
-        key: "Tdense"
-        value {
-          list {
-            type: DT_FLOAT
-          }
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 784
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "dense_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: 784
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "sparse_types"
-        value {
-          list {
-          }
+      op {
+        name: "ZerosLike"
+        input_arg {
+          name: "x"
+          type_attr: "T"
         }
-      }
-    }
-    node {
-      name: "x"
-      op: "Identity"
-      input: "ParseExample/ParseExample"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
+        output_arg {
+          name: "y"
+          type_attr: "T"
         }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 784
-              }
-            }
-          }
+        attr {
+          name: "T"
+          type: "type"
         }
       }
     }
+    tags: "serve"
+    tensorflow_version: "1.4.1"
+    tensorflow_git_version: "v1.4.0-19-ga52c8d9b01"
+  }
+  graph_def {
     node {
       name: "Placeholder"
       op: "Placeholder"
@@ -1412,7 +1161,7 @@ meta_graphs {
                 size: -1
               }
               dim {
-                size: 10
+                size: 784
               }
             }
           }
@@ -1432,7 +1181,7 @@ meta_graphs {
               size: -1
             }
             dim {
-              size: 10
+              size: 784
             }
           }
         }
@@ -1767,15 +1516,9 @@ meta_graphs {
       }
     }
     node {
-      name: "init"
-      op: "NoOp"
-      input: "^Variable/Assign"
-      input: "^Variable_1/Assign"
-    }
-    node {
       name: "MatMul"
       op: "MatMul"
-      input: "x"
+      input: "Placeholder"
       input: "Variable/read"
       attr {
         key: "T"
@@ -1839,15 +1582,8 @@ meta_graphs {
       }
     }
     node {
-      name: "y"
-      op: "Softmax"
-      input: "add"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
+      name: "Placeholder_1"
+      op: "Placeholder"
       attr {
         key: "_output_shapes"
         value {
@@ -1863,38 +1599,60 @@ meta_graphs {
           }
         }
       }
-    }
-    node {
-      name: "Log"
-      op: "Log"
-      input: "y"
       attr {
-        key: "T"
+        key: "dtype"
         value {
           type: DT_FLOAT
         }
       }
       attr {
+        key: "shape"
+        value {
+          shape {
+            dim {
+              size: -1
+            }
+            dim {
+              size: 10
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "Rank"
+      op: "Const"
+      attr {
         key: "_output_shapes"
         value {
           list {
             shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 10
-              }
             }
           }
         }
       }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 2
+          }
+        }
+      }
     }
     node {
-      name: "mul"
-      op: "Mul"
-      input: "Placeholder"
-      input: "Log"
+      name: "Shape"
+      op: "Shape"
+      input: "add"
       attr {
         key: "T"
         value {
@@ -1907,27 +1665,27 @@ meta_graphs {
           list {
             shape {
               dim {
-                size: -1
-              }
-              dim {
-                size: 10
+                size: 2
               }
             }
           }
         }
       }
+      attr {
+        key: "out_type"
+        value {
+          type: DT_INT32
+        }
+      }
     }
     node {
-      name: "Const"
+      name: "Rank_1"
       op: "Const"
       attr {
         key: "_output_shapes"
         value {
           list {
             shape {
-              dim {
-                size: 2
-              }
             }
           }
         }
@@ -1944,20 +1702,16 @@ meta_graphs {
           tensor {
             dtype: DT_INT32
             tensor_shape {
-              dim {
-                size: 2
-              }
             }
-            tensor_content: "\000\000\000\000\001\000\000\000"
+            int_val: 2
           }
         }
       }
     }
     node {
-      name: "Sum"
-      op: "Sum"
-      input: "mul"
-      input: "Const"
+      name: "Shape_1"
+      op: "Shape"
+      input: "add"
       attr {
         key: "T"
         value {
@@ -1965,11 +1719,27 @@ meta_graphs {
         }
       }
       attr {
-        key: "Tidx"
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 2
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "out_type"
         value {
           type: DT_INT32
         }
       }
+    }
+    node {
+      name: "Sub/y"
+      op: "Const"
       attr {
         key: "_output_shapes"
         value {
@@ -1980,20 +1750,32 @@ meta_graphs {
         }
       }
       attr {
-        key: "keep_dims"
+        key: "dtype"
         value {
-          b: false
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 1
+          }
         }
       }
     }
     node {
-      name: "Neg"
-      op: "Neg"
-      input: "Sum"
+      name: "Sub"
+      op: "Sub"
+      input: "Rank_1"
+      input: "Sub/y"
       attr {
         key: "T"
         value {
-          type: DT_FLOAT
+          type: DT_INT32
         }
       }
       attr {
@@ -2007,46 +1789,51 @@ meta_graphs {
       }
     }
     node {
-      name: "gradients/Shape"
-      op: "Const"
+      name: "Slice/begin"
+      op: "Pack"
+      input: "Sub"
       attr {
-        key: "_output_shapes"
+        key: "N"
         value {
-          list {
-            shape {
-              dim {
-              }
-            }
-          }
+          i: 1
         }
       }
       attr {
-        key: "dtype"
+        key: "T"
         value {
           type: DT_INT32
         }
       }
       attr {
-        key: "value"
+        key: "_output_shapes"
         value {
-          tensor {
-            dtype: DT_INT32
-            tensor_shape {
+          list {
+            shape {
               dim {
+                size: 1
               }
             }
           }
         }
       }
+      attr {
+        key: "axis"
+        value {
+          i: 0
+        }
+      }
     }
     node {
-      name: "gradients/Const"
+      name: "Slice/size"
       op: "Const"
       attr {
         key: "_output_shapes"
         value {
           list {
             shape {
+              dim {
+                size: 1
+              }
             }
           }
         }
@@ -2054,50 +1841,40 @@ meta_graphs {
       attr {
         key: "dtype"
         value {
-          type: DT_FLOAT
+          type: DT_INT32
         }
       }
       attr {
         key: "value"
         value {
           tensor {
-            dtype: DT_FLOAT
+            dtype: DT_INT32
             tensor_shape {
+              dim {
+                size: 1
+              }
             }
-            float_val: 1.0
+            int_val: 1
           }
         }
       }
     }
     node {
-      name: "gradients/Fill"
-      op: "Fill"
-      input: "gradients/Shape"
-      input: "gradients/Const"
+      name: "Slice"
+      op: "Slice"
+      input: "Shape_1"
+      input: "Slice/begin"
+      input: "Slice/size"
       attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_output_shapes"
+        key: "Index"
         value {
-          list {
-            shape {
-            }
-          }
+          type: DT_INT32
         }
       }
-    }
-    node {
-      name: "gradients/Neg_grad/Neg"
-      op: "Neg"
-      input: "gradients/Fill"
       attr {
         key: "T"
         value {
-          type: DT_FLOAT
+          type: DT_INT32
         }
       }
       attr {
@@ -2105,13 +1882,16 @@ meta_graphs {
         value {
           list {
             shape {
+              dim {
+                size: 1
+              }
             }
           }
         }
       }
     }
     node {
-      name: "gradients/Sum_grad/Reshape/shape"
+      name: "concat/values_0"
       op: "Const"
       attr {
         key: "_output_shapes"
@@ -2119,7 +1899,7 @@ meta_graphs {
           list {
             shape {
               dim {
-                size: 2
+                size: 1
               }
             }
           }
@@ -2138,55 +1918,66 @@ meta_graphs {
             dtype: DT_INT32
             tensor_shape {
               dim {
-                size: 2
+                size: 1
               }
             }
-            tensor_content: "\001\000\000\000\001\000\000\000"
+            int_val: -1
           }
         }
       }
     }
     node {
-      name: "gradients/Sum_grad/Reshape"
-      op: "Reshape"
-      input: "gradients/Neg_grad/Neg"
-      input: "gradients/Sum_grad/Reshape/shape"
+      name: "concat/axis"
+      op: "Const"
       attr {
-        key: "T"
+        key: "_output_shapes"
         value {
-          type: DT_FLOAT
+          list {
+            shape {
+            }
+          }
         }
       }
       attr {
-        key: "Tshape"
+        key: "dtype"
         value {
           type: DT_INT32
         }
       }
       attr {
-        key: "_output_shapes"
+        key: "value"
         value {
-          list {
-            shape {
-              dim {
-                size: 1
-              }
-              dim {
-                size: 1
-              }
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
             }
+            int_val: 0
           }
         }
       }
     }
     node {
-      name: "gradients/Sum_grad/Shape"
-      op: "Shape"
-      input: "mul"
+      name: "concat"
+      op: "ConcatV2"
+      input: "concat/values_0"
+      input: "Slice"
+      input: "concat/axis"
+      attr {
+        key: "N"
+        value {
+          i: 2
+        }
+      }
       attr {
         key: "T"
         value {
-          type: DT_FLOAT
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "Tidx"
+        value {
+          type: DT_INT32
         }
       }
       attr {
@@ -2201,18 +1992,12 @@ meta_graphs {
           }
         }
       }
-      attr {
-        key: "out_type"
-        value {
-          type: DT_INT32
-        }
-      }
     }
     node {
-      name: "gradients/Sum_grad/Tile"
-      op: "Tile"
-      input: "gradients/Sum_grad/Reshape"
-      input: "gradients/Sum_grad/Shape"
+      name: "Reshape"
+      op: "Reshape"
+      input: "add"
+      input: "concat"
       attr {
         key: "T"
         value {
@@ -2220,7 +2005,7 @@ meta_graphs {
         }
       }
       attr {
-        key: "Tmultiples"
+        key: "Tshape"
         value {
           type: DT_INT32
         }
@@ -2234,7 +2019,7 @@ meta_graphs {
                 size: -1
               }
               dim {
-                size: 10
+                size: -1
               }
             }
           }
@@ -2242,38 +2027,39 @@ meta_graphs {
       }
     }
     node {
-      name: "gradients/mul_grad/Shape"
-      op: "Shape"
-      input: "Placeholder"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
+      name: "Rank_2"
+      op: "Const"
       attr {
         key: "_output_shapes"
         value {
           list {
             shape {
-              dim {
-                size: 2
-              }
             }
           }
         }
       }
       attr {
-        key: "out_type"
+        key: "dtype"
         value {
           type: DT_INT32
         }
       }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 2
+          }
+        }
+      }
     }
     node {
-      name: "gradients/mul_grad/Shape_1"
+      name: "Shape_2"
       op: "Shape"
-      input: "Log"
+      input: "Placeholder_1"
       attr {
         key: "T"
         value {
@@ -2300,43 +2086,44 @@ meta_graphs {
       }
     }
     node {
-      name: "gradients/mul_grad/BroadcastGradientArgs"
-      op: "BroadcastGradientArgs"
-      input: "gradients/mul_grad/Shape"
-      input: "gradients/mul_grad/Shape_1"
+      name: "Sub_1/y"
+      op: "Const"
       attr {
-        key: "T"
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
         value {
           type: DT_INT32
         }
       }
       attr {
-        key: "_output_shapes"
+        key: "value"
         value {
-          list {
-            shape {
-              dim {
-                size: -1
-              }
-            }
-            shape {
-              dim {
-                size: -1
-              }
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
             }
+            int_val: 1
           }
         }
       }
     }
     node {
-      name: "gradients/mul_grad/mul"
-      op: "Mul"
-      input: "gradients/Sum_grad/Tile"
-      input: "Log"
+      name: "Sub_1"
+      op: "Sub"
+      input: "Rank_2"
+      input: "Sub_1/y"
       attr {
         key: "T"
         value {
-          type: DT_FLOAT
+          type: DT_INT32
         }
       }
       attr {
@@ -2344,30 +2131,23 @@ meta_graphs {
         value {
           list {
             shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 10
-              }
             }
           }
         }
       }
     }
     node {
-      name: "gradients/mul_grad/Sum"
-      op: "Sum"
-      input: "gradients/mul_grad/mul"
-      input: "gradients/mul_grad/BroadcastGradientArgs"
+      name: "Slice_1/begin"
+      op: "Pack"
+      input: "Sub_1"
       attr {
-        key: "T"
+        key: "N"
         value {
-          type: DT_FLOAT
+          i: 1
         }
       }
       attr {
-        key: "Tidx"
+        key: "T"
         value {
           type: DT_INT32
         }
@@ -2377,60 +2157,72 @@ meta_graphs {
         value {
           list {
             shape {
-              unknown_rank: true
+              dim {
+                size: 1
+              }
             }
           }
         }
       }
       attr {
-        key: "keep_dims"
+        key: "axis"
         value {
-          b: false
+          i: 0
         }
       }
     }
     node {
-      name: "gradients/mul_grad/Reshape"
-      op: "Reshape"
-      input: "gradients/mul_grad/Sum"
-      input: "gradients/mul_grad/Shape"
+      name: "Slice_1/size"
+      op: "Const"
       attr {
-        key: "T"
+        key: "_output_shapes"
         value {
-          type: DT_FLOAT
+          list {
+            shape {
+              dim {
+                size: 1
+              }
+            }
+          }
         }
       }
       attr {
-        key: "Tshape"
+        key: "dtype"
         value {
           type: DT_INT32
         }
       }
       attr {
-        key: "_output_shapes"
+        key: "value"
         value {
-          list {
-            shape {
-              dim {
-                size: -1
-              }
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
               dim {
-                size: 10
+                size: 1
               }
             }
+            int_val: 1
           }
         }
       }
     }
     node {
-      name: "gradients/mul_grad/mul_1"
-      op: "Mul"
-      input: "Placeholder"
-      input: "gradients/Sum_grad/Tile"
+      name: "Slice_1"
+      op: "Slice"
+      input: "Shape_2"
+      input: "Slice_1/begin"
+      input: "Slice_1/size"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
       attr {
         key: "T"
         value {
-          type: DT_FLOAT
+          type: DT_INT32
         }
       }
       attr {
@@ -2439,10 +2231,7 @@ meta_graphs {
           list {
             shape {
               dim {
-                size: -1
-              }
-              dim {
-                size: 10
+                size: 1
               }
             }
           }
@@ -2450,44 +2239,113 @@ meta_graphs {
       }
     }
     node {
-      name: "gradients/mul_grad/Sum_1"
-      op: "Sum"
-      input: "gradients/mul_grad/mul_1"
-      input: "gradients/mul_grad/BroadcastGradientArgs:1"
+      name: "concat_1/values_0"
+      op: "Const"
       attr {
-        key: "T"
+        key: "_output_shapes"
         value {
-          type: DT_FLOAT
+          list {
+            shape {
+              dim {
+                size: 1
+              }
+            }
+          }
         }
       }
       attr {
-        key: "Tidx"
+        key: "dtype"
         value {
           type: DT_INT32
         }
       }
       attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: -1
+          }
+        }
+      }
+    }
+    node {
+      name: "concat_1/axis"
+      op: "Const"
+      attr {
         key: "_output_shapes"
         value {
           list {
             shape {
-              unknown_rank: true
             }
           }
         }
       }
       attr {
-        key: "keep_dims"
+        key: "dtype"
         value {
-          b: false
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 0
+          }
+        }
+      }
+    }
+    node {
+      name: "concat_1"
+      op: "ConcatV2"
+      input: "concat_1/values_0"
+      input: "Slice_1"
+      input: "concat_1/axis"
+      attr {
+        key: "N"
+        value {
+          i: 2
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "Tidx"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 2
+              }
+            }
+          }
         }
       }
     }
     node {
-      name: "gradients/mul_grad/Reshape_1"
+      name: "Reshape_1"
       op: "Reshape"
-      input: "gradients/mul_grad/Sum_1"
-      input: "gradients/mul_grad/Shape_1"
+      input: "Placeholder_1"
+      input: "concat_1"
       attr {
         key: "T"
         value {
@@ -2509,7 +2367,7 @@ meta_graphs {
                 size: -1
               }
               dim {
-                size: 10
+                size: -1
               }
             }
           }
@@ -2517,16 +2375,10 @@ meta_graphs {
       }
     }
     node {
-      name: "gradients/mul_grad/tuple/group_deps"
-      op: "NoOp"
-      input: "^gradients/mul_grad/Reshape"
-      input: "^gradients/mul_grad/Reshape_1"
-    }
-    node {
-      name: "gradients/mul_grad/tuple/control_dependency"
-      op: "Identity"
-      input: "gradients/mul_grad/Reshape"
-      input: "^gradients/mul_grad/tuple/group_deps"
+      name: "SoftmaxCrossEntropyWithLogits"
+      op: "SoftmaxCrossEntropyWithLogits"
+      input: "Reshape"
+      input: "Reshape_1"
       attr {
         key: "T"
         value {
@@ -2534,73 +2386,127 @@ meta_graphs {
         }
       }
       attr {
-        key: "_class"
+        key: "_output_shapes"
         value {
           list {
-            s: "loc:@gradients/mul_grad/Reshape"
+            shape {
+              dim {
+                size: -1
+              }
+            }
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: -1
+              }
+            }
           }
         }
       }
+    }
+    node {
+      name: "Sub_2/y"
+      op: "Const"
       attr {
         key: "_output_shapes"
         value {
           list {
             shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 10
-              }
             }
           }
         }
       }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 1
+          }
+        }
+      }
     }
     node {
-      name: "gradients/mul_grad/tuple/control_dependency_1"
-      op: "Identity"
-      input: "gradients/mul_grad/Reshape_1"
-      input: "^gradients/mul_grad/tuple/group_deps"
+      name: "Sub_2"
+      op: "Sub"
+      input: "Rank"
+      input: "Sub_2/y"
       attr {
         key: "T"
         value {
-          type: DT_FLOAT
+          type: DT_INT32
         }
       }
       attr {
-        key: "_class"
+        key: "_output_shapes"
         value {
           list {
-            s: "loc:@gradients/mul_grad/Reshape_1"
+            shape {
+            }
           }
         }
       }
+    }
+    node {
+      name: "Slice_2/begin"
+      op: "Const"
       attr {
         key: "_output_shapes"
         value {
           list {
             shape {
               dim {
-                size: -1
+                size: 1
               }
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
               dim {
-                size: 10
+                size: 1
               }
             }
+            int_val: 0
           }
         }
       }
     }
     node {
-      name: "gradients/Log_grad/Reciprocal"
-      op: "Reciprocal"
-      input: "y"
-      input: "^gradients/mul_grad/tuple/control_dependency_1"
+      name: "Slice_2/size"
+      op: "Pack"
+      input: "Sub_2"
+      attr {
+        key: "N"
+        value {
+          i: 1
+        }
+      }
       attr {
         key: "T"
         value {
-          type: DT_FLOAT
+          type: DT_INT32
         }
       }
       attr {
@@ -2609,25 +2515,35 @@ meta_graphs {
           list {
             shape {
               dim {
-                size: -1
-              }
-              dim {
-                size: 10
+                size: 1
               }
             }
           }
         }
       }
+      attr {
+        key: "axis"
+        value {
+          i: 0
+        }
+      }
     }
     node {
-      name: "gradients/Log_grad/mul"
-      op: "Mul"
-      input: "gradients/mul_grad/tuple/control_dependency_1"
-      input: "gradients/Log_grad/Reciprocal"
+      name: "Slice_2"
+      op: "Slice"
+      input: "Shape"
+      input: "Slice_2/begin"
+      input: "Slice_2/size"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
       attr {
         key: "T"
         value {
-          type: DT_FLOAT
+          type: DT_INT32
         }
       }
       attr {
@@ -2638,19 +2554,16 @@ meta_graphs {
               dim {
                 size: -1
               }
-              dim {
-                size: 10
-              }
             }
           }
         }
       }
     }
     node {
-      name: "gradients/y_grad/mul"
-      op: "Mul"
-      input: "gradients/Log_grad/mul"
-      input: "y"
+      name: "Reshape_2"
+      op: "Reshape"
+      input: "SoftmaxCrossEntropyWithLogits"
+      input: "Slice_2"
       attr {
         key: "T"
         value {
@@ -2658,6 +2571,12 @@ meta_graphs {
         }
       }
       attr {
+        key: "Tshape"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
         key: "_output_shapes"
         value {
           list {
@@ -2665,16 +2584,13 @@ meta_graphs {
               dim {
                 size: -1
               }
-              dim {
-                size: 10
-              }
             }
           }
         }
       }
     }
     node {
-      name: "gradients/y_grad/Sum/reduction_indices"
+      name: "Const"
       op: "Const"
       attr {
         key: "_output_shapes"
@@ -2704,16 +2620,16 @@ meta_graphs {
                 size: 1
               }
             }
-            int_val: 1
+            int_val: 0
           }
         }
       }
     }
     node {
-      name: "gradients/y_grad/Sum"
-      op: "Sum"
-      input: "gradients/y_grad/mul"
-      input: "gradients/y_grad/Sum/reduction_indices"
+      name: "Mean"
+      op: "Mean"
+      input: "Reshape_2"
+      input: "Const"
       attr {
         key: "T"
         value {
@@ -2731,9 +2647,6 @@ meta_graphs {
         value {
           list {
             shape {
-              dim {
-                size: -1
-              }
             }
           }
         }
@@ -2746,7 +2659,7 @@ meta_graphs {
       }
     }
     node {
-      name: "gradients/y_grad/Reshape/shape"
+      name: "gradients/Shape"
       op: "Const"
       attr {
         key: "_output_shapes"
@@ -2754,7 +2667,6 @@ meta_graphs {
           list {
             shape {
               dim {
-                size: 2
               }
             }
           }
@@ -2773,19 +2685,47 @@ meta_graphs {
             dtype: DT_INT32
             tensor_shape {
               dim {
-                size: 2
               }
             }
-            tensor_content: "\377\377\377\377\001\000\000\000"
           }
         }
       }
     }
     node {
-      name: "gradients/y_grad/Reshape"
-      op: "Reshape"
-      input: "gradients/y_grad/Sum"
-      input: "gradients/y_grad/Reshape/shape"
+      name: "gradients/Const"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 1.0
+          }
+        }
+      }
+    }
+    node {
+      name: "gradients/Fill"
+      op: "Fill"
+      input: "gradients/Shape"
+      input: "gradients/Const"
       attr {
         key: "T"
         value {
@@ -2793,32 +2733,56 @@ meta_graphs {
         }
       }
       attr {
-        key: "Tshape"
+        key: "_output_shapes"
         value {
-          type: DT_INT32
+          list {
+            shape {
+            }
+          }
         }
       }
+    }
+    node {
+      name: "gradients/Mean_grad/Reshape/shape"
+      op: "Const"
       attr {
         key: "_output_shapes"
         value {
           list {
             shape {
               dim {
-                size: -1
+                size: 1
               }
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
               dim {
                 size: 1
               }
             }
+            int_val: 1
           }
         }
       }
     }
     node {
-      name: "gradients/y_grad/sub"
-      op: "Sub"
-      input: "gradients/Log_grad/mul"
-      input: "gradients/y_grad/Reshape"
+      name: "gradients/Mean_grad/Reshape"
+      op: "Reshape"
+      input: "gradients/Fill"
+      input: "gradients/Mean_grad/Reshape/shape"
       attr {
         key: "T"
         value {
@@ -2826,26 +2790,58 @@ meta_graphs {
         }
       }
       attr {
+        key: "Tshape"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
         key: "_output_shapes"
         value {
           list {
             shape {
               dim {
-                size: -1
+                size: 1
               }
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "gradients/Mean_grad/Shape"
+      op: "Shape"
+      input: "Reshape_2"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
               dim {
-                size: 10
+                size: 1
               }
             }
           }
         }
       }
+      attr {
+        key: "out_type"
+        value {
+          type: DT_INT32
+        }
+      }
     }
     node {
-      name: "gradients/y_grad/mul_1"
-      op: "Mul"
-      input: "gradients/y_grad/sub"
-      input: "y"
+      name: "gradients/Mean_grad/Tile"
+      op: "Tile"
+      input: "gradients/Mean_grad/Reshape"
+      input: "gradients/Mean_grad/Shape"
       attr {
         key: "T"
         value {
@@ -2853,6 +2849,12 @@ meta_graphs {
         }
       }
       attr {
+        key: "Tmultiples"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
         key: "_output_shapes"
         value {
           list {
@@ -2860,18 +2862,15 @@ meta_graphs {
               dim {
                 size: -1
               }
-              dim {
-                size: 10
-              }
             }
           }
         }
       }
     }
     node {
-      name: "gradients/add_grad/Shape"
+      name: "gradients/Mean_grad/Shape_1"
       op: "Shape"
-      input: "MatMul"
+      input: "Reshape_2"
       attr {
         key: "T"
         value {
@@ -2884,7 +2883,7 @@ meta_graphs {
           list {
             shape {
               dim {
-                size: 2
+                size: 1
               }
             }
           }
@@ -2898,7 +2897,7 @@ meta_graphs {
       }
     }
     node {
-      name: "gradients/add_grad/Shape_1"
+      name: "gradients/Mean_grad/Shape_2"
       op: "Const"
       attr {
         key: "_output_shapes"
@@ -2906,7 +2905,6 @@ meta_graphs {
           list {
             shape {
               dim {
-                size: 1
               }
             }
           }
@@ -2925,23 +2923,21 @@ meta_graphs {
             dtype: DT_INT32
             tensor_shape {
               dim {
-                size: 1
               }
             }
-            int_val: 10
           }
         }
       }
     }
     node {
-      name: "gradients/add_grad/BroadcastGradientArgs"
-      op: "BroadcastGradientArgs"
-      input: "gradients/add_grad/Shape"
-      input: "gradients/add_grad/Shape_1"
+      name: "gradients/Mean_grad/Const"
+      op: "Const"
       attr {
-        key: "T"
+        key: "_class"
         value {
-          type: DT_INT32
+          list {
+            s: "loc:@gradients/Mean_grad/Shape_1"
+          }
         }
       }
       attr {
@@ -2950,27 +2946,42 @@ meta_graphs {
           list {
             shape {
               dim {
-                size: -1
+                size: 1
               }
             }
-            shape {
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
               dim {
-                size: -1
+                size: 1
               }
             }
+            int_val: 0
           }
         }
       }
     }
     node {
-      name: "gradients/add_grad/Sum"
-      op: "Sum"
-      input: "gradients/y_grad/mul_1"
-      input: "gradients/add_grad/BroadcastGradientArgs"
+      name: "gradients/Mean_grad/Prod"
+      op: "Prod"
+      input: "gradients/Mean_grad/Shape_1"
+      input: "gradients/Mean_grad/Const"
       attr {
         key: "T"
         value {
-          type: DT_FLOAT
+          type: DT_INT32
         }
       }
       attr {
@@ -2980,11 +2991,18 @@ meta_graphs {
         }
       }
       attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@gradients/Mean_grad/Shape_1"
+          }
+        }
+      }
+      attr {
         key: "_output_shapes"
         value {
           list {
             shape {
-              unknown_rank: true
             }
           }
         }
@@ -2997,20 +3015,14 @@ meta_graphs {
       }
     }
     node {
-      name: "gradients/add_grad/Reshape"
-      op: "Reshape"
-      input: "gradients/add_grad/Sum"
-      input: "gradients/add_grad/Shape"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
+      name: "gradients/Mean_grad/Const_1"
+      op: "Const"
       attr {
-        key: "Tshape"
+        key: "_class"
         value {
-          type: DT_INT32
+          list {
+            s: "loc:@gradients/Mean_grad/Shape_1"
+          }
         }
       }
       attr {
@@ -3019,25 +3031,42 @@ meta_graphs {
           list {
             shape {
               dim {
-                size: -1
+                size: 1
               }
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
               dim {
-                size: 10
+                size: 1
               }
             }
+            int_val: 0
           }
         }
       }
     }
     node {
-      name: "gradients/add_grad/Sum_1"
-      op: "Sum"
-      input: "gradients/y_grad/mul_1"
-      input: "gradients/add_grad/BroadcastGradientArgs:1"
+      name: "gradients/Mean_grad/Prod_1"
+      op: "Prod"
+      input: "gradients/Mean_grad/Shape_2"
+      input: "gradients/Mean_grad/Const_1"
       attr {
         key: "T"
         value {
-          type: DT_FLOAT
+          type: DT_INT32
         }
       }
       attr {
@@ -3047,11 +3076,18 @@ meta_graphs {
         }
       }
       attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@gradients/Mean_grad/Shape_1"
+          }
+        }
+      }
+      attr {
         key: "_output_shapes"
         value {
           list {
             shape {
-              unknown_rank: true
             }
           }
         }
@@ -3064,57 +3100,59 @@ meta_graphs {
       }
     }
     node {
-      name: "gradients/add_grad/Reshape_1"
-      op: "Reshape"
-      input: "gradients/add_grad/Sum_1"
-      input: "gradients/add_grad/Shape_1"
+      name: "gradients/Mean_grad/Maximum/y"
+      op: "Const"
       attr {
-        key: "T"
+        key: "_class"
         value {
-          type: DT_FLOAT
+          list {
+            s: "loc:@gradients/Mean_grad/Shape_1"
+          }
         }
       }
       attr {
-        key: "Tshape"
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
         value {
           type: DT_INT32
         }
       }
       attr {
-        key: "_output_shapes"
+        key: "value"
         value {
-          list {
-            shape {
-              dim {
-                size: 10
-              }
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
             }
+            int_val: 1
           }
         }
       }
     }
     node {
-      name: "gradients/add_grad/tuple/group_deps"
-      op: "NoOp"
-      input: "^gradients/add_grad/Reshape"
-      input: "^gradients/add_grad/Reshape_1"
-    }
-    node {
-      name: "gradients/add_grad/tuple/control_dependency"
-      op: "Identity"
-      input: "gradients/add_grad/Reshape"
-      input: "^gradients/add_grad/tuple/group_deps"
+      name: "gradients/Mean_grad/Maximum"
+      op: "Maximum"
+      input: "gradients/Mean_grad/Prod_1"
+      input: "gradients/Mean_grad/Maximum/y"
       attr {
         key: "T"
         value {
-          type: DT_FLOAT
+          type: DT_INT32
         }
       }
       attr {
         key: "_class"
         value {
           list {
-            s: "loc:@gradients/add_grad/Reshape"
+            s: "loc:@gradients/Mean_grad/Shape_1"
           }
         }
       }
@@ -3123,33 +3161,27 @@ meta_graphs {
         value {
           list {
             shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 10
-              }
             }
           }
         }
       }
     }
     node {
-      name: "gradients/add_grad/tuple/control_dependency_1"
-      op: "Identity"
-      input: "gradients/add_grad/Reshape_1"
-      input: "^gradients/add_grad/tuple/group_deps"
+      name: "gradients/Mean_grad/floordiv"
+      op: "FloorDiv"
+      input: "gradients/Mean_grad/Prod"
+      input: "gradients/Mean_grad/Maximum"
       attr {
         key: "T"
         value {
-          type: DT_FLOAT
+          type: DT_INT32
         }
       }
       attr {
         key: "_class"
         value {
           list {
-            s: "loc:@gradients/add_grad/Reshape_1"
+            s: "loc:@gradients/Mean_grad/Shape_1"
           }
         }
       }
@@ -3158,58 +3190,65 @@ meta_graphs {
         value {
           list {
             shape {
-              dim {
-                size: 10
-              }
             }
           }
         }
       }
     }
     node {
-      name: "gradients/MatMul_grad/MatMul"
-      op: "MatMul"
-      input: "gradients/add_grad/tuple/control_dependency"
-      input: "Variable/read"
+      name: "gradients/Mean_grad/Cast"
+      op: "Cast"
+      input: "gradients/Mean_grad/floordiv"
       attr {
-        key: "T"
+        key: "DstT"
         value {
           type: DT_FLOAT
         }
       }
       attr {
+        key: "SrcT"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
         key: "_output_shapes"
         value {
           list {
             shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 784
-              }
             }
           }
         }
       }
+    }
+    node {
+      name: "gradients/Mean_grad/truediv"
+      op: "RealDiv"
+      input: "gradients/Mean_grad/Tile"
+      input: "gradients/Mean_grad/Cast"
       attr {
-        key: "transpose_a"
+        key: "T"
         value {
-          b: false
+          type: DT_FLOAT
         }
       }
       attr {
-        key: "transpose_b"
+        key: "_output_shapes"
         value {
-          b: true
+          list {
+            shape {
+              dim {
+                size: -1
+              }
+            }
+          }
         }
       }
     }
     node {
-      name: "gradients/MatMul_grad/MatMul_1"
-      op: "MatMul"
-      input: "x"
-      input: "gradients/add_grad/tuple/control_dependency"
+      name: "gradients/Reshape_2_grad/Shape"
+      op: "Shape"
+      input: "SoftmaxCrossEntropyWithLogits"
       attr {
         key: "T"
         value {
@@ -3222,39 +3261,24 @@ meta_graphs {
           list {
             shape {
               dim {
-                size: 784
-              }
-              dim {
-                size: 10
+                size: 1
               }
             }
           }
         }
       }
       attr {
-        key: "transpose_a"
-        value {
-          b: true
-        }
-      }
-      attr {
-        key: "transpose_b"
+        key: "out_type"
         value {
-          b: false
+          type: DT_INT32
         }
       }
     }
     node {
-      name: "gradients/MatMul_grad/tuple/group_deps"
-      op: "NoOp"
-      input: "^gradients/MatMul_grad/MatMul"
-      input: "^gradients/MatMul_grad/MatMul_1"
-    }
-    node {
-      name: "gradients/MatMul_grad/tuple/control_dependency"
-      op: "Identity"
-      input: "gradients/MatMul_grad/MatMul"
-      input: "^gradients/MatMul_grad/tuple/group_deps"
+      name: "gradients/Reshape_2_grad/Reshape"
+      op: "Reshape"
+      input: "gradients/Mean_grad/truediv"
+      input: "gradients/Reshape_2_grad/Shape"
       attr {
         key: "T"
         value {
@@ -3262,11 +3286,9 @@ meta_graphs {
         }
       }
       attr {
-        key: "_class"
+        key: "Tshape"
         value {
-          list {
-            s: "loc:@gradients/MatMul_grad/MatMul"
-          }
+          type: DT_INT32
         }
       }
       attr {
@@ -3277,19 +3299,15 @@ meta_graphs {
               dim {
                 size: -1
               }
-              dim {
-                size: 784
-              }
             }
           }
         }
       }
     }
     node {
-      name: "gradients/MatMul_grad/tuple/control_dependency_1"
-      op: "Identity"
-      input: "gradients/MatMul_grad/MatMul_1"
-      input: "^gradients/MatMul_grad/tuple/group_deps"
+      name: "gradients/zeros_like"
+      op: "ZerosLike"
+      input: "SoftmaxCrossEntropyWithLogits:1"
       attr {
         key: "T"
         value {
@@ -3297,23 +3315,15 @@ meta_graphs {
         }
       }
       attr {
-        key: "_class"
-        value {
-          list {
-            s: "loc:@gradients/MatMul_grad/MatMul_1"
-          }
-        }
-      }
-      attr {
         key: "_output_shapes"
         value {
           list {
             shape {
               dim {
-                size: 784
+                size: -1
               }
               dim {
-                size: 10
+                size: -1
               }
             }
           }
@@ -3321,7 +3331,7 @@ meta_graphs {
       }
     }
     node {
-      name: "GradientDescent/learning_rate"
+      name: "gradients/SoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim"
       op: "Const"
       attr {
         key: "_output_shapes"
@@ -3335,27 +3345,26 @@ meta_graphs {
       attr {
         key: "dtype"
         value {
-          type: DT_FLOAT
+          type: DT_INT32
         }
       }
       attr {
         key: "value"
         value {
           tensor {
-            dtype: DT_FLOAT
+            dtype: DT_INT32
             tensor_shape {
             }
-            float_val: 0.00999999977648
+            int_val: -1
           }
         }
       }
     }
     node {
-      name: "GradientDescent/update_Variable/ApplyGradientDescent"
-      op: "ApplyGradientDescent"
-      input: "Variable"
-      input: "GradientDescent/learning_rate"
-      input: "gradients/MatMul_grad/tuple/control_dependency_1"
+      name: "gradients/SoftmaxCrossEntropyWithLogits_grad/ExpandDims"
+      op: "ExpandDims"
+      input: "gradients/Reshape_2_grad/Reshape"
+      input: "gradients/SoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim"
       attr {
         key: "T"
         value {
@@ -3363,11 +3372,9 @@ meta_graphs {
         }
       }
       attr {
-        key: "_class"
+        key: "Tdim"
         value {
-          list {
-            s: "loc:@Variable"
-          }
+          type: DT_INT32
         }
       }
       attr {
@@ -3376,28 +3383,21 @@ meta_graphs {
           list {
             shape {
               dim {
-                size: 784
+                size: -1
               }
               dim {
-                size: 10
+                size: 1
               }
             }
           }
         }
       }
-      attr {
-        key: "use_locking"
-        value {
-          b: false
-        }
-      }
     }
     node {
-      name: "GradientDescent/update_Variable_1/ApplyGradientDescent"
-      op: "ApplyGradientDescent"
-      input: "Variable_1"
-      input: "GradientDescent/learning_rate"
-      input: "gradients/add_grad/tuple/control_dependency_1"
+      name: "gradients/SoftmaxCrossEntropyWithLogits_grad/mul"
+      op: "Mul"
+      input: "gradients/SoftmaxCrossEntropyWithLogits_grad/ExpandDims"
+      input: "SoftmaxCrossEntropyWithLogits:1"
       attr {
         key: "T"
         value {
@@ -3405,73 +3405,87 @@ meta_graphs {
         }
       }
       attr {
-        key: "_class"
+        key: "_output_shapes"
         value {
           list {
-            s: "loc:@Variable_1"
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: -1
+              }
+            }
           }
         }
       }
+    }
+    node {
+      name: "gradients/Reshape_grad/Shape"
+      op: "Shape"
+      input: "add"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
       attr {
         key: "_output_shapes"
         value {
           list {
             shape {
               dim {
-                size: 10
+                size: 2
               }
             }
           }
         }
       }
       attr {
-        key: "use_locking"
+        key: "out_type"
         value {
-          b: false
+          type: DT_INT32
         }
       }
     }
     node {
-      name: "GradientDescent"
-      op: "NoOp"
-      input: "^GradientDescent/update_Variable/ApplyGradientDescent"
-      input: "^GradientDescent/update_Variable_1/ApplyGradientDescent"
-    }
-    node {
-      name: "TopKV2/k"
-      op: "Const"
+      name: "gradients/Reshape_grad/Reshape"
+      op: "Reshape"
+      input: "gradients/SoftmaxCrossEntropyWithLogits_grad/mul"
+      input: "gradients/Reshape_grad/Shape"
       attr {
-        key: "_output_shapes"
+        key: "T"
         value {
-          list {
-            shape {
-            }
-          }
+          type: DT_FLOAT
         }
       }
       attr {
-        key: "dtype"
+        key: "Tshape"
         value {
           type: DT_INT32
         }
       }
       attr {
-        key: "value"
+        key: "_output_shapes"
         value {
-          tensor {
-            dtype: DT_INT32
-            tensor_shape {
+          list {
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 10
+              }
             }
-            int_val: 10
           }
         }
       }
     }
     node {
-      name: "TopKV2"
-      op: "TopKV2"
-      input: "y"
-      input: "TopKV2/k"
+      name: "gradients/add_grad/Shape"
+      op: "Shape"
+      input: "MatMul"
       attr {
         key: "T"
         value {
@@ -3484,32 +3498,21 @@ meta_graphs {
           list {
             shape {
               dim {
-                size: -1
-              }
-              dim {
-                size: 10
-              }
-            }
-            shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 10
+                size: 2
               }
             }
           }
         }
       }
       attr {
-        key: "sorted"
+        key: "out_type"
         value {
-          b: true
+          type: DT_INT32
         }
       }
     }
     node {
-      name: "Const_1"
+      name: "gradients/add_grad/Shape_1"
       op: "Const"
       attr {
         key: "_output_shapes"
@@ -3517,7 +3520,7 @@ meta_graphs {
           list {
             shape {
               dim {
-                size: 10
+                size: 1
               }
             }
           }
@@ -3526,133 +3529,207 @@ meta_graphs {
       attr {
         key: "dtype"
         value {
-          type: DT_STRING
+          type: DT_INT32
         }
       }
       attr {
         key: "value"
         value {
           tensor {
-            dtype: DT_STRING
+            dtype: DT_INT32
             tensor_shape {
               dim {
-                size: 10
+                size: 1
               }
             }
-            string_val: "0"
-            string_val: "1"
-            string_val: "2"
-            string_val: "3"
-            string_val: "4"
-            string_val: "5"
-            string_val: "6"
-            string_val: "7"
-            string_val: "8"
-            string_val: "9"
+            int_val: 10
           }
         }
       }
     }
     node {
-      name: "index_to_string/Size"
-      op: "Const"
+      name: "gradients/add_grad/BroadcastGradientArgs"
+      op: "BroadcastGradientArgs"
+      input: "gradients/add_grad/Shape"
+      input: "gradients/add_grad/Shape_1"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
       attr {
         key: "_output_shapes"
         value {
           list {
             shape {
+              dim {
+                size: -1
+              }
+            }
+            shape {
+              dim {
+                size: -1
+              }
             }
           }
         }
       }
+    }
+    node {
+      name: "gradients/add_grad/Sum"
+      op: "Sum"
+      input: "gradients/Reshape_grad/Reshape"
+      input: "gradients/add_grad/BroadcastGradientArgs"
       attr {
-        key: "dtype"
+        key: "T"
         value {
-          type: DT_INT32
+          type: DT_FLOAT
         }
       }
       attr {
-        key: "value"
+        key: "Tidx"
         value {
-          tensor {
-            dtype: DT_INT32
-            tensor_shape {
-            }
-            int_val: 10
-          }
+          type: DT_INT32
         }
       }
-    }
-    node {
-      name: "index_to_string/range/start"
-      op: "Const"
       attr {
         key: "_output_shapes"
         value {
           list {
             shape {
+              unknown_rank: true
             }
           }
         }
       }
       attr {
-        key: "dtype"
+        key: "keep_dims"
+        value {
+          b: false
+        }
+      }
+    }
+    node {
+      name: "gradients/add_grad/Reshape"
+      op: "Reshape"
+      input: "gradients/add_grad/Sum"
+      input: "gradients/add_grad/Shape"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "Tshape"
         value {
           type: DT_INT32
         }
       }
       attr {
-        key: "value"
+        key: "_output_shapes"
         value {
-          tensor {
-            dtype: DT_INT32
-            tensor_shape {
+          list {
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 10
+              }
             }
-            int_val: 0
           }
         }
       }
     }
     node {
-      name: "index_to_string/range/delta"
-      op: "Const"
+      name: "gradients/add_grad/Sum_1"
+      op: "Sum"
+      input: "gradients/Reshape_grad/Reshape"
+      input: "gradients/add_grad/BroadcastGradientArgs:1"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "Tidx"
+        value {
+          type: DT_INT32
+        }
+      }
       attr {
         key: "_output_shapes"
         value {
           list {
             shape {
+              unknown_rank: true
             }
           }
         }
       }
       attr {
-        key: "dtype"
+        key: "keep_dims"
+        value {
+          b: false
+        }
+      }
+    }
+    node {
+      name: "gradients/add_grad/Reshape_1"
+      op: "Reshape"
+      input: "gradients/add_grad/Sum_1"
+      input: "gradients/add_grad/Shape_1"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "Tshape"
         value {
           type: DT_INT32
         }
       }
       attr {
-        key: "value"
+        key: "_output_shapes"
         value {
-          tensor {
-            dtype: DT_INT32
-            tensor_shape {
+          list {
+            shape {
+              dim {
+                size: 10
+              }
             }
-            int_val: 1
           }
         }
       }
     }
     node {
-      name: "index_to_string/range"
-      op: "Range"
-      input: "index_to_string/range/start"
-      input: "index_to_string/Size"
-      input: "index_to_string/range/delta"
+      name: "gradients/add_grad/tuple/group_deps"
+      op: "NoOp"
+      input: "^gradients/add_grad/Reshape"
+      input: "^gradients/add_grad/Reshape_1"
+    }
+    node {
+      name: "gradients/add_grad/tuple/control_dependency"
+      op: "Identity"
+      input: "gradients/add_grad/Reshape"
+      input: "^gradients/add_grad/tuple/group_deps"
       attr {
-        key: "Tidx"
+        key: "T"
         value {
-          type: DT_INT32
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@gradients/add_grad/Reshape"
+          }
         }
       }
       attr {
@@ -3661,6 +3738,9 @@ meta_graphs {
           list {
             shape {
               dim {
+                size: -1
+              }
+              dim {
                 size: 10
               }
             }
@@ -3669,19 +3749,22 @@ meta_graphs {
       }
     }
     node {
-      name: "index_to_string/ToInt64"
-      op: "Cast"
-      input: "index_to_string/range"
+      name: "gradients/add_grad/tuple/control_dependency_1"
+      op: "Identity"
+      input: "gradients/add_grad/Reshape_1"
+      input: "^gradients/add_grad/tuple/group_deps"
       attr {
-        key: "DstT"
+        key: "T"
         value {
-          type: DT_INT64
+          type: DT_FLOAT
         }
       }
       attr {
-        key: "SrcT"
+        key: "_class"
         value {
-          type: DT_INT32
+          list {
+            s: "loc:@gradients/add_grad/Reshape_1"
+          }
         }
       }
       attr {
@@ -3698,111 +3781,207 @@ meta_graphs {
       }
     }
     node {
-      name: "index_to_string"
-      op: "HashTableV2"
+      name: "gradients/MatMul_grad/MatMul"
+      op: "MatMul"
+      input: "gradients/add_grad/tuple/control_dependency"
+      input: "Variable/read"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
       attr {
         key: "_output_shapes"
         value {
           list {
             shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 784
+              }
             }
           }
         }
       }
       attr {
-        key: "container"
+        key: "transpose_a"
         value {
-          s: ""
+          b: false
         }
       }
       attr {
-        key: "key_dtype"
+        key: "transpose_b"
         value {
-          type: DT_INT64
+          b: true
         }
       }
+    }
+    node {
+      name: "gradients/MatMul_grad/MatMul_1"
+      op: "MatMul"
+      input: "Placeholder"
+      input: "gradients/add_grad/tuple/control_dependency"
       attr {
-        key: "shared_name"
+        key: "T"
         value {
-          s: ""
+          type: DT_FLOAT
         }
       }
       attr {
-        key: "use_node_name_sharing"
+        key: "_output_shapes"
         value {
-          b: false
+          list {
+            shape {
+              dim {
+                size: 784
+              }
+              dim {
+                size: 10
+              }
+            }
+          }
         }
       }
       attr {
-        key: "value_dtype"
+        key: "transpose_a"
         value {
-          type: DT_STRING
+          b: true
+        }
+      }
+      attr {
+        key: "transpose_b"
+        value {
+          b: false
         }
       }
     }
     node {
-      name: "index_to_string/Const"
-      op: "Const"
+      name: "gradients/MatMul_grad/tuple/group_deps"
+      op: "NoOp"
+      input: "^gradients/MatMul_grad/MatMul"
+      input: "^gradients/MatMul_grad/MatMul_1"
+    }
+    node {
+      name: "gradients/MatMul_grad/tuple/control_dependency"
+      op: "Identity"
+      input: "gradients/MatMul_grad/MatMul"
+      input: "^gradients/MatMul_grad/tuple/group_deps"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@gradients/MatMul_grad/MatMul"
+          }
+        }
+      }
       attr {
         key: "_output_shapes"
         value {
           list {
             shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 784
+              }
             }
           }
         }
       }
+    }
+    node {
+      name: "gradients/MatMul_grad/tuple/control_dependency_1"
+      op: "Identity"
+      input: "gradients/MatMul_grad/MatMul_1"
+      input: "^gradients/MatMul_grad/tuple/group_deps"
       attr {
-        key: "dtype"
+        key: "T"
         value {
-          type: DT_STRING
+          type: DT_FLOAT
         }
       }
       attr {
-        key: "value"
+        key: "_class"
         value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
+          list {
+            s: "loc:@gradients/MatMul_grad/MatMul_1"
+          }
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 784
+              }
+              dim {
+                size: 10
+              }
             }
-            string_val: "UNK"
           }
         }
       }
     }
     node {
-      name: "index_to_string/table_init"
-      op: "InitializeTableV2"
-      input: "index_to_string"
-      input: "index_to_string/ToInt64"
-      input: "Const_1"
+      name: "GradientDescent/learning_rate"
+      op: "Const"
       attr {
-        key: "Tkey"
+        key: "_output_shapes"
         value {
-          type: DT_INT64
+          list {
+            shape {
+            }
+          }
         }
       }
       attr {
-        key: "Tval"
+        key: "dtype"
         value {
-          type: DT_STRING
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 0.5
+          }
         }
       }
     }
     node {
-      name: "ToInt64"
-      op: "Cast"
-      input: "TopKV2:1"
+      name: "GradientDescent/update_Variable/ApplyGradientDescent"
+      op: "ApplyGradientDescent"
+      input: "Variable"
+      input: "GradientDescent/learning_rate"
+      input: "gradients/MatMul_grad/tuple/control_dependency_1"
       attr {
-        key: "DstT"
+        key: "T"
         value {
-          type: DT_INT64
+          type: DT_FLOAT
         }
       }
       attr {
-        key: "SrcT"
+        key: "_class"
         value {
-          type: DT_INT32
+          list {
+            s: "loc:@Variable"
+          }
         }
       }
       attr {
@@ -3811,7 +3990,7 @@ meta_graphs {
           list {
             shape {
               dim {
-                size: -1
+                size: 784
               }
               dim {
                 size: 10
@@ -3820,23 +3999,31 @@ meta_graphs {
           }
         }
       }
+      attr {
+        key: "use_locking"
+        value {
+          b: false
+        }
+      }
     }
     node {
-      name: "index_to_string_Lookup"
-      op: "LookupTableFindV2"
-      input: "index_to_string"
-      input: "ToInt64"
-      input: "index_to_string/Const"
+      name: "GradientDescent/update_Variable_1/ApplyGradientDescent"
+      op: "ApplyGradientDescent"
+      input: "Variable_1"
+      input: "GradientDescent/learning_rate"
+      input: "gradients/add_grad/tuple/control_dependency_1"
       attr {
-        key: "Tin"
+        key: "T"
         value {
-          type: DT_INT64
+          type: DT_FLOAT
         }
       }
       attr {
-        key: "Tout"
+        key: "_class"
         value {
-          type: DT_STRING
+          list {
+            s: "loc:@Variable_1"
+          }
         }
       }
       attr {
@@ -3845,15 +4032,30 @@ meta_graphs {
           list {
             shape {
               dim {
-                size: -1
-              }
-              dim {
                 size: 10
               }
             }
           }
         }
       }
+      attr {
+        key: "use_locking"
+        value {
+          b: false
+        }
+      }
+    }
+    node {
+      name: "GradientDescent"
+      op: "NoOp"
+      input: "^GradientDescent/update_Variable/ApplyGradientDescent"
+      input: "^GradientDescent/update_Variable_1/ApplyGradientDescent"
+    }
+    node {
+      name: "init"
+      op: "NoOp"
+      input: "^Variable/Assign"
+      input: "^Variable_1/Assign"
     }
     node {
       name: "ArgMax/dimension"
@@ -3888,7 +4090,7 @@ meta_graphs {
     node {
       name: "ArgMax"
       op: "ArgMax"
-      input: "y"
+      input: "add"
       input: "ArgMax/dimension"
       attr {
         key: "T"
@@ -3954,7 +4156,7 @@ meta_graphs {
     node {
       name: "ArgMax_1"
       op: "ArgMax"
-      input: "Placeholder"
+      input: "Placeholder_1"
       input: "ArgMax_1/dimension"
       attr {
         key: "T"
@@ -4012,7 +4214,7 @@ meta_graphs {
       }
     }
     node {
-      name: "Cast"
+      name: "Cast_1"
       op: "Cast"
       input: "Equal"
       attr {
@@ -4041,7 +4243,7 @@ meta_graphs {
       }
     }
     node {
-      name: "Const_2"
+      name: "Const_1"
       op: "Const"
       attr {
         key: "_output_shapes"
@@ -4077,10 +4279,10 @@ meta_graphs {
       }
     }
     node {
-      name: "Mean"
+      name: "Mean_1"
       op: "Mean"
-      input: "Cast"
-      input: "Const_2"
+      input: "Cast_1"
+      input: "Const_1"
       attr {
         key: "T"
         value {
@@ -4110,16 +4312,6 @@ meta_graphs {
       }
     }
     node {
-      name: "init_all_tables"
-      op: "NoOp"
-      input: "^index_to_string/table_init"
-    }
-    node {
-      name: "legacy_init_op"
-      op: "NoOp"
-      input: "^init_all_tables"
-    }
-    node {
       name: "save/Const"
       op: "Const"
       attr {
@@ -4174,7 +4366,7 @@ meta_graphs {
             dtype: DT_STRING
             tensor_shape {
             }
-            string_val: "_temp_8390c48b96834292ab57050d0ae6959e/part"
+            string_val: "_temp_6ca9fa5171ed4237a2fbcc27277e2864/part"
           }
         }
       }
@@ -4783,22 +4975,6 @@ meta_graphs {
     version: V2
   }
   collection_def {
-    key: "legacy_init_op"
-    value {
-      node_list {
-        value: "legacy_init_op"
-      }
-    }
-  }
-  collection_def {
-    key: "table_initializer"
-    value {
-      node_list {
-        value: "index_to_string/table_init"
-      }
-    }
-  }
-  collection_def {
     key: "train_op"
     value {
       node_list {
@@ -4810,8 +4986,8 @@ meta_graphs {
     key: "trainable_variables"
     value {
       bytes_list {
-        value: "\n\nVariable:0\022\017Variable/Assign\032\017Variable/read:0"
-        value: "\n\014Variable_1:0\022\021Variable_1/Assign\032\021Variable_1/read:0"
+        value: "\n\nVariable:0\022\017Variable/Assign\032\017Variable/read:02\007zeros:0"
+        value: "\n\014Variable_1:0\022\021Variable_1/Assign\032\021Variable_1/read:02\tzeros_1:0"
       }
     }
   }
@@ -4819,18 +4995,18 @@ meta_graphs {
     key: "variables"
     value {
       bytes_list {
-        value: "\n\nVariable:0\022\017Variable/Assign\032\017Variable/read:0"
-        value: "\n\014Variable_1:0\022\021Variable_1/Assign\032\021Variable_1/read:0"
+        value: "\n\nVariable:0\022\017Variable/Assign\032\017Variable/read:02\007zeros:0"
+        value: "\n\014Variable_1:0\022\021Variable_1/Assign\032\021Variable_1/read:02\tzeros_1:0"
       }
     }
   }
   signature_def {
-    key: "predict_images"
+    key: "serving_default"
     value {
       inputs {
-        key: "images"
+        key: "x"
         value {
-          name: "x:0"
+          name: "Placeholder:0"
           dtype: DT_FLOAT
           tensor_shape {
             dim {
@@ -4843,9 +5019,9 @@ meta_graphs {
         }
       }
       outputs {
-        key: "scores"
+        key: "y"
         value {
-          name: "y:0"
+          name: "add:0"
           dtype: DT_FLOAT
           tensor_shape {
             dim {
@@ -4860,50 +5036,4 @@ meta_graphs {
       method_name: "tensorflow/serving/predict"
     }
   }
-  signature_def {
-    key: "serving_default"
-    value {
-      inputs {
-        key: "inputs"
-        value {
-          name: "tf_example:0"
-          dtype: DT_STRING
-          tensor_shape {
-            unknown_rank: true
-          }
-        }
-      }
-      outputs {
-        key: "classes"
-        value {
-          name: "index_to_string_Lookup:0"
-          dtype: DT_STRING
-          tensor_shape {
-            dim {
-              size: -1
-            }
-            dim {
-              size: 10
-            }
-          }
-        }
-      }
-      outputs {
-        key: "scores"
-        value {
-          name: "TopKV2:0"
-          dtype: DT_FLOAT
-          tensor_shape {
-            dim {
-              size: -1
-            }
-            dim {
-              size: 10
-            }
-          }
-        }
-      }
-      method_name: "tensorflow/serving/classify"
-    }
-  }
 }
diff --git a/searchlib/src/test/files/integration/tensorflow/mnist_softmax/saved/variables/variables.data-00000-of-00001 b/searchlib/src/test/files/integration/tensorflow/mnist_softmax/saved/variables/variables.data-00000-of-00001
new file mode 100644
index 00000000000..8474aa0a04c
--- /dev/null
+++ b/searchlib/src/test/files/integration/tensorflow/mnist_softmax/saved/variables/variables.data-00000-of-00001
diff --git a/searchlib/src/test/files/integration/tensorflow/mnist_softmax/saved/variables/variables.index b/searchlib/src/test/files/integration/tensorflow/mnist_softmax/saved/variables/variables.index
new file mode 100644
index 00000000000..cfcdac20409
--- /dev/null
+++ b/searchlib/src/test/files/integration/tensorflow/mnist_softmax/saved/variables/variables.index
diff --git a/searchlib/src/test/files/integration/tensorflow/model1/variables/variables.data-00000-of-00001 b/searchlib/src/test/files/integration/tensorflow/model1/variables/variables.data-00000-of-00001
deleted file mode 100644
index ba71c21fbe1..00000000000
--- a/searchlib/src/test/files/integration/tensorflow/model1/variables/variables.data-00000-of-00001
+++ /dev/null
diff --git a/searchlib/src/test/files/integration/tensorflow/model1/variables/variables.index b/searchlib/src/test/files/integration/tensorflow/model1/variables/variables.index
deleted file mode 100644
index 84f4593515a..00000000000
--- a/searchlib/src/test/files/integration/tensorflow/model1/variables/variables.index
+++ /dev/null